Patched memory merging

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.4",
+	"version": "1.6.5",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/memory.ts

@@ -4,7 +4,7 @@ import {AiTool} from './tools.ts';
 import {KDPoint, KDTree} from './kd-tree.ts';
 import {escapeRegex} from '@ztimson/utils';
 
-const MERGE_THRESHOLD = 0.88;
+const MERGE_THRESHOLD = 0.12;
 const PENDING_HEADING = '## Pending';
 const GENERIC_TEMPLATE = `# {{Title}}
 
@@ -191,13 +191,13 @@ export type MemoryOptions = {
 }
 
 export class MemoryManager {
-	private recentlyTouched = new Map<string, number>();
-
+	private mergeLock: Promise<any> = Promise.resolve();
 	private queues = new Map<string, {
 		dirty: boolean,
 		request: {abort?: () => void} | null,
 		task: Promise<void>,
 	}>();
+	private recentlyTouched = new Map<string, number>();
 
 	tools = {
 		forget: (memories: Memory[] | MemoryCache): AiTool => ({
@@ -351,10 +351,6 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return memories.map(m => ({name: m.name, description: m.description}));
 	}
 
-	/** Find the nearest node above the similarity threshold and fold the smaller/less-connected one into
-	 * the other. Journals are exempt — they're partitioned by date, not topic, and merging across weeks
-	 * would wreck the timeline. Returns 'merged' if `node` absorbed another (caller should re-run the doc
-	 * agent), 'absorbed' if `node` itself got folded away (caller should stop touching it), or null. */
 	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
 		if (!node.embedding?.length || node.name.startsWith('Journal/')) return null;
 		const store = this.access(memories);
@@ -404,8 +400,9 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			do {
 				entry.dirty = false;
 				await this.docAgent(current, store.list, options, entry);
-				const merged = await this.checkMerge(current, memories, options);
-				if (merged) { current = merged; entry.dirty = true; }
+				this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
+				const merged = await this.mergeLock;
+				if(merged) current = merged;
 			} while (entry.dirty);
 		})().finally(() => {
 			this.queues.delete(key);
@@ -432,11 +429,9 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
 
 Use this loose structure, adapting headings to what the content needs:
-
-# Title
-## Summary
-## Details
-## Related
+\`\`\`markdown
+${GENERIC_TEMPLATE}
+\`\`\`
 
 Rules:
 - Contradictions: newer facts always win — delete outdated statements entirely
@@ -483,11 +478,9 @@ ${currentBody}
 			system: `You are a knowledge base editor merging two overlapping Obsidian documents into one. Newer facts win on contradiction.
 
 Structure loosely:
-
-# Title
-## Summary
-## Details
-## Related
+\`\`\`markdown
+${GENERIC_TEMPLATE}
+\`\`\`
 
 Combine both documents, resolve duplication and contradictions.
 
@@ -617,16 +610,16 @@ ${stripHeader(b.content)}
 			touched.push(node);
 		}
 
-		for (const node of touched) {
+		await Promise.all(touched.map(async node => {
 			const [e] = await this.llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
 			if (e) node.embedding = e.embedding;
 			this.touch(node.name);
-		}
+		}));
 
 		if (touched.length) {
 			store.commit();
 			(pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`;
-			await Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
+			Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
 		} else {
 			(pending as any).content = 'Nothing worth remembering.';
 		}