Better memory management

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.2",
+	"version": "1.6.3",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/memory.ts

@@ -2,9 +2,10 @@ import {MemoryNode, rebuildGraph} from './helpers.ts';
 import {LLMRequest, LLMMessage} from './llm.ts';
 import {AiTool} from './tools.ts';
 import {KDPoint, KDTree} from './kd-tree.ts';
+import {escapeRegex} from '@ztimson/utils';
 
-const FACTS_HEADING = '## Facts';
-
+const MERGE_THRESHOLD = 0.88;
+const PENDING_HEADING = '## Pending';
 const GENERIC_TEMPLATE = `# {{Title}}
 
 ## Summary
@@ -32,6 +33,11 @@ type FactBucket = {
 	facts: string[];
 }
 
+type FactAgentResult = {
+	buckets: FactBucket[];
+	journal: string;
+}
+
 function dedupeFacts(facts: string[]): string[] {
 	const seen = new Map<string, string>();
 	for (const f of facts) {
@@ -251,14 +257,13 @@ ${m.content}
 		return new MemoryAccessor(memories);
 	}
 
-	private appendFacts(node: Memory, facts: string[]): void {
+	private stage(node: Memory, block: string): void {
 		this.ensureDoc(node);
 		const body = stripHeader(node.content);
-		const bullets = facts.map(f => `- ${f}`).join('\n');
-		const idx = body.indexOf(FACTS_HEADING);
+		const idx = body.indexOf(PENDING_HEADING);
 		const newBody = idx === -1
-			? `${body.trimEnd()}\n\n${FACTS_HEADING}\n${bullets}\n`
-			: `${body.slice(0, idx + FACTS_HEADING.length)}\n${bullets}${body.slice(idx + FACTS_HEADING.length)}`;
+			? `${body.trimEnd()}\n\n${PENDING_HEADING}\n${block}\n`
+			: `${body.slice(0, idx + PENDING_HEADING.length)}\n${block}${body.slice(idx + PENDING_HEADING.length)}`;
 		node.content = this.touchHeader(node, newBody);
 	}
 
@@ -268,16 +273,30 @@ ${m.content}
 		node.content = this.touchHeader(node, `# ${title}\n`);
 	}
 
-	private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest, weekKey: string): Promise<FactBucket[]> {
+	private sanitizeDescription(text: string): string {
+		return (text ?? '').replace(/\s+/g, ' ').trim().slice(0, 240);
+	}
+
+	private relink(memories: Memory[], from: string, to: string): void {
+		const pattern = new RegExp(`\\[\\[${escapeRegex(from)}\\]\\]`, 'g');
+		for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`);
+	}
+
+	private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest): Promise<FactAgentResult> {
 		const ghosts = store.ghosts();
 
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
 			system: `You are a fact extractor to build obsidian knowledge vaults.
-Analyze this conversation and extract facts worth remembering long-term.
+Analyze this conversation and produce two things: a journal recap and any long-term facts worth filing.
 
-Rules:
+Journal recap rules:
+- Write a short day-to-day recap of this conversation: what was discussed/worked on, decisions still in flux, the user's state/mood if mentioned, general context
+- Write something whenever there was any meaningful activity, even if nothing below qualifies as a hard fact
+- Leave it empty only for trivial/empty exchanges (greetings, pleasantries, nothing happened)
+
+Fact bucket rules:
 - Always extract facts that the user explicitly told you to remember
 - ONLY extract current facts the USER explicitly stated about themselves, their work, projects or decisions that were MADE during this conversation
 - DO NOT extract greetings, pleasantries, or generic exchanges
@@ -286,19 +305,20 @@ Rules:
 - If nothing worth remembering was said, return an empty buckets array
 
 When extracting facts, you MUST also decide the exact destination path:
-- Reuse node names (including ghost) as much as possible IF the facts belongs there
+- Reuse node names (including ghost) as much as possible IF the fact belongs there
 - All information primarily about the user should go under "People/User"
-- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide) — you are not limited to any fixed list of collections, use whatever fits
-- For journal entries, use "Journal"
+- NEVER use a generic label like "Technical Issue", "Technical Decision", "Technical Requirement", "Bug", "Feature Request" as a subject — these fragment the vault. Every fact belongs to a concrete subject: a person, project, or system.
+- A bug report, its investigation, and its eventual fix are the SAME topic — reuse the one node for the subject, don't fork a new one per event
+- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide), named after the subject, not the event — you are not limited to any fixed list of collections
 
 Available nodes:
-- Journal
-${this.listNodes(store.list).filter(n => !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
+${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
 ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			schema: {
+				journal: {type: 'string', description: 'Short day-to-day recap of this conversation, see rules above. Empty string if nothing happened.', required: false},
 				buckets: {type: 'array', description: 'Groups of facts to remember, each assigned to a different node. Return an empty array if there is nothing worth storing in an obsidian vault', items: {
 						type: 'object', items: {
-							subject: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide"), or "Journal"', required: true},
+							subject: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
 							facts: {
 								type: 'array',
 								description: 'Facts to store at this destination',
@@ -311,15 +331,17 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		});
 
 		const buckets = new Map<string, string[]>();
-		for(const bucket of response.buckets ?? []) {
-			const subject = bucket.subject.trim().toLowerCase() === 'journal'
-				? `Journal/${weekKey}` : bucket.subject.trim();
+		for (const bucket of response.buckets ?? []) {
+			const subject = bucket.subject.trim();
 			const facts = buckets.get(subject) ?? [];
 			facts.push(...dedupeFacts(bucket.facts));
 			buckets.set(subject, facts);
 		}
 
-		return buckets.entries().toArray().map(([subject, facts]) => ({subject, facts}));
+		return {
+			buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
+			journal: (response.journal ?? '').trim(),
+		};
 	}
 
 	private getWeekMonday(date: Date = new Date()): string {
@@ -334,6 +356,34 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return memories.map(m => ({name: m.name, description: m.description}));
 	}
 
+	/** Find the nearest node above the similarity threshold and fold the smaller/less-connected one into
+	 * the other. Journals are exempt — they're partitioned by date, not topic, and merging across weeks
+	 * would wreck the timeline. Returns 'merged' if `node` absorbed another (caller should re-run the doc
+	 * agent), 'absorbed' if `node` itself got folded away (caller should stop touching it), or null. */
+	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<'merged' | 'absorbed' | null> {
+		if (!node.embedding?.length || node.name.startsWith('Journal/')) return null;
+		const store = this.access(memories);
+
+		let closest: Memory | null = null, closestDist = Infinity;
+		for (const other of store.list) {
+			if (other.name === node.name || other.name.startsWith('Journal/') || !other.embedding?.length) continue;
+			const d = cosineDistance(node.embedding, other.embedding);
+			if (d < closestDist) { closestDist = d; closest = other; }
+		}
+		if (!closest || closestDist > threshold) return null;
+
+		const weight = (m: Memory) => m.content.length + (m.links.length + m.backlinks.length) * 200;
+		const [primary, secondary] = weight(node) >= weight(closest) ? [node, closest] : [closest, node];
+
+		this.stage(primary, `### Merged from [[${secondary.name}]]\n\n${stripHeader(secondary.content)}`);
+		this.relink(store.list, secondary.name, primary.name);
+		store.forget(secondary.name);
+
+		if (primary === node) return 'merged';
+		this.reconcile(primary, memories, options).catch(() => {});
+		return 'absorbed';
+	}
+
 	private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
 		const key = node.name;
 		const existing = this.queues.get(key);
@@ -350,6 +400,9 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			do {
 				entry.dirty = false;
 				await this.docAgent(node, store.list, options, entry);
+				const merge = await this.checkMerge(node, memories, options);
+				if (merge === 'absorbed') break;
+				if (merge === 'merged') entry.dirty = true;
 			} while (entry.dirty);
 		})().finally(() => {
 			this.queues.delete(key);
@@ -360,6 +413,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 
 	private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
 		const currentBody = stripHeader(node.content);
+		const isJournal = node.name.startsWith('Journal/');
 		let update;
 		try {
 			for (let i = 0; i < 2 && !update?.content; i++) {
@@ -367,24 +421,27 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 					model: options.model,
 					temperature: 0.3,
 					schema: {
-						description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true},
+						description: {type: 'string', description: 'One factual sentence describing what this document contains. Never a thought process, reasoning, or meta-commentary about the task.', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
 					system: `You are a knowledge base editor maintaining one document in an Obsidian-style vault.
 
-If the document has a "${FACTS_HEADING}" section, integrate every bullet under it into the appropriate part of the document, then remove the "${FACTS_HEADING}" section entirely. If there is no such section, just tidy the document per the rules below.
+If the document has a "${PENDING_HEADING}" section, it holds new material to integrate — bullet facts, a journal entry, or an entire document merged in from elsewhere (marked "### Merged from [[X]]" or "### {date}"). Fold all of it into the appropriate part of the document, resolving overlap and duplication, then remove the "${PENDING_HEADING}" section entirely. If there is no such section, just tidy the document per the rules below.
 
-Structure: follow this generic shape loosely, adapting section names/order to what the content actually needs (e.g. journal-style docs may want a timeline instead of "Details"):
+Structure: follow this generic shape loosely, adapting section names/order to what the content actually needs:
 \`\`\`markdown
 ${GENERIC_TEMPLATE}
 \`\`\`
 
+${isJournal
+						? `This document is a journal — a chronological day-to-day log. Keep entries ordered as a timeline under "### {date}" headings. NEVER delete or "resolve away" older entries in favor of newer ones — history stays, only clean up grammar/clarity within an entry.`
+						: `Resolve contradictions: newer facts always win — delete the outdated statement entirely, never keep both.`}
+
 Formatting rules:
 - Use Obsidian-style markdown: # headings, **bold** for emphasis, bullet & numbered lists for grouped 1D data, tables for 2D data
 - Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]]
 - Create links for specific entities (person, place, project, program) and abstract concepts, but skip generics (car, red, dog)
 - Keep the document concise, factual, and human-readable
-- Resolve contradictions: newer facts always win — delete the outdated statement entirely, never keep both
 - Do not add frontmatter blocks, filler, preamble, or AI commentary
 
 Other nodes in the vault (link to these instead of duplicating their content):
@@ -406,7 +463,7 @@ ${currentBody}
 		}
 
 		if (!update?.content) return;
-		node.description = node.name !== 'People/User' ? update.description : 'All information about the current user';
+		node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
 		node.content = this.touchHeader(node, update.content);
 		const [e] = await this.llm.embedding(node.content);
 		if (e) node.embedding = e.embedding;
@@ -419,7 +476,11 @@ ${currentBody}
 		for (const line of match[1].split('\n')) {
 			const i = line.indexOf(':');
 			if (i === -1) continue;
-			fm.set(line.slice(0, i).trim(), line.slice(i + 1).trim());
+			const key = line.slice(0, i).trim();
+			const raw = line.slice(i + 1).trim();
+			let value = raw;
+			try { value = JSON.parse(raw); } catch { /* legacy unquoted value, keep raw */ }
+			fm.set(key, value);
 		}
 		return {fm, body: match[2]};
 	}
@@ -433,7 +494,7 @@ ${currentBody}
 	}
 
 	private writeFrontmatter(fm: Map<string, string>, body: string): string {
-		const lines = [...fm.entries()].map(([k, v]) => `${k}: ${v}`);
+		const lines = [...fm.entries()].map(([k, v]) => `${k}: ${JSON.stringify(String(v).replace(/\s+/g, ' ').trim())}`);
 		return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`;
 	}
 
@@ -498,20 +559,34 @@ ${currentBody}
 		history.push(pending);
 
 		const store = this.access(memories);
-		const buckets = await this.factAgent(conversation, store, options, this.getWeekMonday());
+		const {buckets, journal} = await this.factAgent(conversation, store, options);
 		const touched: Memory[] = [];
 
+		if (journal) {
+			const journalName = `Journal/${this.getWeekMonday()}`;
+			let jnode = store.find(journalName);
+			if (!jnode) {
+				jnode = {name: journalName, description: '', content: '', embedding: [], links: [], backlinks: []};
+				store.list.push(jnode);
+			}
+			this.stage(jnode, `### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+			touched.push(jnode);
+		}
+
 		for (const {subject, facts} of buckets) {
 			let node = store.find(subject);
 			if (!node) {
 				node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []};
 				store.list.push(node);
 			}
-			this.appendFacts(node, facts);
+			this.stage(node, facts.map(f => `- ${f}`).join('\n'));
+			touched.push(node);
+		}
+
+		for (const node of touched) {
 			const [e] = await this.llm.embedding(node.content);
 			if (e) node.embedding = e.embedding;
 			this.touch(node.name);
-			touched.push(node);
 		}
 
 		if (touched.length) {
@@ -526,9 +601,9 @@ ${currentBody}
 		return touched;
 	}
 
-	async reconcileVault(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
+	async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
 		const store = this.access(memories);
-		const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(FACTS_HEADING));
+		const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING));
 		await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
 		store.commit();
 	}