Keep message progress on abort

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.3",
+	"version": "1.6.6",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/ai.ts

@@ -4,7 +4,7 @@ import { Audio } from './audio.ts';
 import {Vision} from './vision.ts';
 
 export type AbortablePromise<T> = Promise<T> & {
-	abort: () => any
+	abort: (keep?: boolean) => any
 };
 
 export type AiOptions = {

src/llm.ts

@@ -265,7 +265,7 @@ class LLM {
 		};
 	}
 
-	private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: (() => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
+	private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: ((keep?: boolean) => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
 		return agents.map(a => {
 			const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`;
 			return {
@@ -397,11 +397,13 @@ ${a.system}`,
 		if(!this.models[m]) throw new Error(`Model does not exist: ${m}`);
 		let request: AbortablePromise<string> | null = null;
 		let aborted = false;
-		const nestedAborts: (() => void)[] = [];
+		let keepOnAbort = true;
-		const abort = () => {
+		const nestedAborts: ((keep?: boolean) => void)[] = [];
+		const abort = (keep = true) => {
 			aborted = true;
-			request?.abort?.();
+			keepOnAbort = keep;
-			nestedAborts.forEach(a => a());
+			request?.abort?.(keep);
+			nestedAborts.forEach(a => a(keep));
 		};
 
 		let promise: any;
@@ -411,9 +413,25 @@ ${a.system}`,
 			let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
 			const prompts: string[] = [];
 			let history = options.history || [];
+			const historyStart = history.length;
 			const files = options.files || [];
 			if(message || files.length) history.push({role: 'user', content: message || '', timestamp: Date.now()});
 
+			// Accumulate streamed text so it can be committed to history if aborted mid-generation
+			let partialText = '';
+			const onStream = options.stream;
+			const stream = (chunk: {text?: string, tool?: string, done?: true}) => {
+				if(chunk.text) partialText += chunk.text;
+				return onStream?.(chunk);
+			};
+
+			/** Commit (keep) or discard this turn's progress on abort, then throw */
+			const abortNow = (): never => {
+				if(keepOnAbort) { if(partialText) history.push({role: 'assistant', content: partialText, timestamp: Date.now()}); }
+				else history.splice(historyStart, history.length - historyStart);
+				throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
+			};
+
 			// MCP
 			const mcp = options.mcp || this.ai.options?.llm?.mcp;
 			if(mcp?.length) {
@@ -441,8 +459,8 @@ ${a.system}`,
 				const mems = mem.memory instanceof MemoryCache ? mem.memory.memories : mem.memory;
 				if(mems.length) {
 					if(mem.inject) {
-						const pool = 15; // candidates considered, cheap since only refs are listed
+						const pool = 15;
-						const budget = mem.maxTokens ?? 2000; // actual content injected
+						const budget = mem.maxTokens ?? 2000;
 						const relevant = await this.memoryManager.recollect(message, mem.memory, pool);
 
 						let used = 0;
@@ -481,7 +499,7 @@ Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')}
 				}
 			}
 
-			if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
+			if(aborted) abortNow();
 
 			const lastMsg = history[history.length - 1];
 			if(files.length && lastMsg?.role === 'user') lastMsg.files = files;
@@ -500,11 +518,17 @@ Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')}
 			const toolTimings = new Map<string, {duration: number, tps: number}>();
 			tools = this.wrapToolTiming(tools, toolTimings);
 
-			if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
+			if(aborted) abortNow();
 
 			prompts.unshift(options.system || this.ai.options.llm?.system || '');
-			request = this.models[m].ask('', {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
+			request = this.models[m].ask('', {...options, tools, stream, system: prompts.filter(Boolean).join('\n\n')});
-			let resp = await request;
+			let resp: string;
+			try {
+				resp = await request;
+			} catch(err: any) {
+				if(aborted) return abortNow();
+				throw err;
+			}
 
 			// Strip the file injection shim
 			restores.forEach(({msg, content}) => msg.content = content);

src/memory.ts

@@ -4,7 +4,7 @@ import {AiTool} from './tools.ts';
 import {KDPoint, KDTree} from './kd-tree.ts';
 import {escapeRegex} from '@ztimson/utils';
 
-const MERGE_THRESHOLD = 0.88;
+const MERGE_THRESHOLD = 0.12;
 const PENDING_HEADING = '## Pending';
 const GENERIC_TEMPLATE = `# {{Title}}
 
@@ -107,8 +107,8 @@ export class MemoryCache {
 	}
 
 	update(memory: Memory): void {
-		const idx = this.memories.findIndex(m => m.name === memory.name);
+		const existing = this.memories.find(m => m.name === memory.name);
-		if (idx !== -1) this.memories[idx] = memory;
+		if (existing) Object.assign(existing, memory);
 		else this.memories.push(memory);
 		this.rebuild();
 	}
@@ -169,7 +169,7 @@ class MemoryAccessor {
 		const missing = this.list.filter(m => !m.embedding?.length);
 		if (!missing.length) return 0;
 		await Promise.all(missing.map(async node => {
-			const [e] = await llm.embedding(node.content);
+			const [e] = await llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
 			if (e) node.embedding = e.embedding;
 		}));
 		this.commit();
@@ -191,13 +191,13 @@ export type MemoryOptions = {
 }
 
 export class MemoryManager {
-	private recentlyTouched = new Map<string, number>();
+	private mergeLock: Promise<any> = Promise.resolve();
-
 	private queues = new Map<string, {
 		dirty: boolean,
 		request: {abort?: () => void} | null,
 		task: Promise<void>,
 	}>();
+	private recentlyTouched = new Map<string, number>();
 
 	tools = {
 		forget: (memories: Memory[] | MemoryCache): AiTool => ({
@@ -288,42 +288,37 @@ ${m.content}
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
-			system: `You are a fact extractor to build obsidian knowledge vaults.
+			system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce:
-Analyze this conversation and produce two things: a journal recap and any long-term facts worth filing.
 
-Journal recap rules:
+1. Journal recap (single paragraph)
-- Write a short day-to-day recap of this conversation: what was discussed/worked on, decisions still in flux, the user's state/mood if mentioned, general context
+   - "Captains Log" style record keeping
-- Write something whenever there was any meaningful activity, even if nothing below qualifies as a hard fact
+   - What was discussed/worked on, decisions, user's events/state/mood, general context
-- Leave it empty only for trivial/empty exchanges (greetings, pleasantries, nothing happened)
+   - Leave empty only for trivial/empty exchanges/small talk
 
-Fact bucket rules:
+2. Fact buckets
-- Always extract facts that the user explicitly told you to remember
+   - ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation
-- ONLY extract current facts the USER explicitly stated about themselves, their work, projects or decisions that were MADE during this conversation
+   - NEVER extract greetings, pleasantries, or anything the assistant itself said
-- DO NOT extract greetings, pleasantries, or generic exchanges
+   - Extract the final/end state, not deltas
-- DO NOT extract deltas or changes in facts; ONLY the end fact
-- DO NOT extract anything the AI/assistant itself said
-- If nothing worth remembering was said, return an empty buckets array
 
-When extracting facts, you MUST also decide the exact destination path:
+Path assignment rules:
-- Reuse node names (including ghost) as much as possible IF the fact belongs there
+   - Reuse existing node names whenever possible
-- All information primarily about the user should go under "People/User"
+   - Documents should be grouped and named by the root subject
-- NEVER use a generic label like "Technical Issue", "Technical Decision", "Technical Requirement", "Bug", "Feature Request" as a subject — these fragment the vault. Every fact belongs to a concrete subject: a person, project, or system.
+     - Person → People/Name
-- A bug report, its investigation, and its eventual fix are the SAME topic — reuse the one node for the subject, don't fork a new one per event
+     - Project → Projects/Name
-- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide), named after the subject, not the event — you are not limited to any fixed list of collections
+     - Concept → Concepts/Name
+   - A bug report, its investigation, should be nested and attached to the same root subject node
+   - Tickets/one-off tasks → file under the project/name/component they belong to
+   - Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\`
 
 Available nodes:
 ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
 ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			schema: {
-				journal: {type: 'string', description: 'Short day-to-day recap of this conversation, see rules above. Empty string if nothing happened.', required: false},
+				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
-				buckets: {type: 'array', description: 'Groups of facts to remember, each assigned to a different node. Return an empty array if there is nothing worth storing in an obsidian vault', items: {
+				buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
 						type: 'object', items: {
-							subject: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
+							subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
-							facts: {
+							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
-								type: 'array',
-								description: 'Facts to store at this destination',
-								items: {type: 'string', description: 'A single fact'},
-							},
 						},
 					},
 				},
@@ -356,11 +351,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return memories.map(m => ({name: m.name, description: m.description}));
 	}
 
-	/** Find the nearest node above the similarity threshold and fold the smaller/less-connected one into
+	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
-	 * the other. Journals are exempt — they're partitioned by date, not topic, and merging across weeks
-	 * would wreck the timeline. Returns 'merged' if `node` absorbed another (caller should re-run the doc
-	 * agent), 'absorbed' if `node` itself got folded away (caller should stop touching it), or null. */
-	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<'merged' | 'absorbed' | null> {
 		if (!node.embedding?.length || node.name.startsWith('Journal/')) return null;
 		const store = this.access(memories);
 
@@ -372,16 +363,24 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		}
 		if (!closest || closestDist > threshold) return null;
 
-		const weight = (m: Memory) => m.content.length + (m.links.length + m.backlinks.length) * 200;
+		const result = await this.mergeAgent(node, closest, options);
-		const [primary, secondary] = weight(node) >= weight(closest) ? [node, closest] : [closest, node];
+		const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []};
+		merged.content = this.touchHeader(merged, result.content);
+		const [e] = await this.llm.embedding(`${merged.description}\n\n${result.content}`.trim());
+		if (e) merged.embedding = e.embedding;
 
-		this.stage(primary, `### Merged from [[${secondary.name}]]\n\n${stripHeader(secondary.content)}`);
+		this.relink(store.list, node.name, merged.name);
-		this.relink(store.list, secondary.name, primary.name);
+		this.relink(store.list, closest.name, merged.name);
-		store.forget(secondary.name);
 
-		if (primary === node) return 'merged';
+		this.queues.get(closest.name)?.request?.abort?.();
-		this.reconcile(primary, memories, options).catch(() => {});
+		this.queues.delete(closest.name);
-		return 'absorbed';
+
+		store.forget(node.name);
+		store.forget(closest.name);
+		store.list.push(merged);
+		store.commit();
+
+		return merged;
 	}
 
 	private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
@@ -397,12 +396,13 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		this.queues.set(key, entry);
 		const store = this.access(memories);
 		entry.task = (async () => {
+			let current = node;
 			do {
 				entry.dirty = false;
-				await this.docAgent(node, store.list, options, entry);
+				await this.docAgent(current, store.list, options, entry);
-				const merge = await this.checkMerge(node, memories, options);
+				this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
-				if (merge === 'absorbed') break;
+				const merged = await this.mergeLock;
-				if (merge === 'merged') entry.dirty = true;
+				if(merged) current = merged;
 			} while (entry.dirty);
 		})().finally(() => {
 			this.queues.delete(key);
@@ -412,8 +412,8 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 	}
 
 	private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
+		if(!memories.includes(node)) return;
 		const currentBody = stripHeader(node.content);
-		const isJournal = node.name.startsWith('Journal/');
 		let update;
 		try {
 			for (let i = 0; i < 2 && !update?.content; i++) {
@@ -421,30 +421,27 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 					model: options.model,
 					temperature: 0.3,
 					schema: {
-						description: {type: 'string', description: 'One factual sentence describing what this document contains. Never a thought process, reasoning, or meta-commentary about the task.', required: true},
+						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
-					system: `You are a knowledge base editor maintaining one document in an Obsidian-style vault.
+					system: `You are a knowledge base editor maintaining one Obsidian-style document.
 
-If the document has a "${PENDING_HEADING}" section, it holds new material to integrate — bullet facts, a journal entry, or an entire document merged in from elsewhere (marked "### Merged from [[X]]" or "### {date}"). Fold all of it into the appropriate part of the document, resolving overlap and duplication, then remove the "${PENDING_HEADING}" section entirely. If there is no such section, just tidy the document per the rules below.
+If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
 
-Structure: follow this generic shape loosely, adapting section names/order to what the content actually needs:
+Use this loose structure, adapting headings to what the content needs:
 \`\`\`markdown
 ${GENERIC_TEMPLATE}
 \`\`\`
 
-${isJournal
+Rules:
-						? `This document is a journal — a chronological day-to-day log. Keep entries ordered as a timeline under "### {date}" headings. NEVER delete or "resolve away" older entries in favor of newer ones — history stays, only clean up grammar/clarity within an entry.`
+- Contradictions: newer facts always win — delete outdated statements entirely
-						: `Resolve contradictions: newer facts always win — delete the outdated statement entirely, never keep both.`}
+- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
+- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
+- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
+- Keep concise, factual, human-readable
+- NO frontmatter, filler, preamble, or AI commentary
 
-Formatting rules:
+Available nodes to link to (don't duplicate their content):
-- Use Obsidian-style markdown: # headings, **bold** for emphasis, bullet & numbered lists for grouped 1D data, tables for 2D data
-- Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]]
-- Create links for specific entities (person, place, project, program) and abstract concepts, but skip generics (car, red, dog)
-- Keep the document concise, factual, and human-readable
-- Do not add frontmatter blocks, filler, preamble, or AI commentary
-
-Other nodes in the vault (link to these instead of duplicating their content):
 ${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
 
 Current document:
@@ -465,10 +462,40 @@ ${currentBody}
 		if (!update?.content) return;
 		node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
 		node.content = this.touchHeader(node, update.content);
-		const [e] = await this.llm.embedding(node.content);
+		const [e] = await this.llm.embedding(`${node.description}\n\n${update.content}`.trim());
 		if (e) node.embedding = e.embedding;
 	}
 
+	private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> {
+		return this.llm.ask('', {
+			model: options.model,
+			temperature: 0.3,
+			schema: {
+				name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true},
+				description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
+				content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
+			},
+			system: `You are a knowledge base editor merging two overlapping Obsidian documents into one. Newer facts win on contradiction.
+
+Structure loosely:
+\`\`\`markdown
+${GENERIC_TEMPLATE}
+\`\`\`
+
+Combine both documents, resolve duplication and contradictions.
+
+Document A ("${a.name}"):
+\`\`\`markdown
+${stripHeader(a.content)}
+\`\`\`
+
+Document B ("${b.name}"):
+\`\`\`markdown
+${stripHeader(b.content)}
+\`\`\``,
+		});
+	}
+
 	private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
 		const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
 		if (!match) return {fm: new Map(), body: content};
@@ -583,16 +610,16 @@ ${currentBody}
 			touched.push(node);
 		}
 
-		for (const node of touched) {
+		await Promise.all(touched.map(async node => {
-			const [e] = await this.llm.embedding(node.content);
+			const [e] = await this.llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
 			if (e) node.embedding = e.embedding;
 			this.touch(node.name);
-		}
+		}));
 
 		if (touched.length) {
 			store.commit();
 			(pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`;
-			await Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
+			Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
 		} else {
 			(pending as any).content = 'Nothing worth remembering.';
 		}