More memory optimizations

2026-09-25 00:33:20 -04:00
parent b5aec246ac
commit 2921b208da
10 changed files with 367 additions and 395 deletions
@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.7.1",
+	"version": "1.7.2",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",
@@ -1,11 +1,14 @@
 export * from './ai';
 export * from './antrhopic';
 export * from './audio';
 export * from './helpers';
 export * from './llm';
-export * from './memory';
+export * from './memory/graph';
 export * from './memory/kd-tree';
 export * from './memory/memory';
 export * from './memory/memory-state';
 export * from './open-ai';
 export * from './provider';
 export * from './token-pool'
 export * from './tools';
 export * from './vision';
 export * from './utils';
@@ -1,17 +1,19 @@
 import {clean, makeUnique, snakeCase} from '@ztimson/utils';
 import {AbortablePromise, Ai} from './ai.ts';
 import {Anthropic} from './antrhopic.ts';
 import {MemoryCache} from './memory/memory-state.ts';
 import {Memory, MemoryManager, MemoryOptions} from './memory/memory.ts';
 import {OpenAi} from './open-ai.ts';
 import {LLMProvider} from './provider.ts';
 import {AiTool, AiToolArg} from './tools.ts';
 import {fileURLToPath} from 'url';
 import {spawn} from 'node:child_process';
 import {Memory, MemoryCache, MemoryManager, MemoryOptions, stripHeader} from './memory.ts';
 import {mkdtempSync} from 'node:fs';
 import fs from 'node:fs/promises';
 import {tmpdir} from 'node:os';
 import {dirname, join, basename, extname} from 'path';
 import { PDFParse } from 'pdf-parse';
 import {stripHeader} from './utils.ts';
 const MAX_AGENT_DEPTH = 5;
 const PDF_OCR_PAGE_THRESHOLD = 12; // above this many pages, OCR scanned pages instead of feeding images to the model
@@ -501,7 +503,7 @@ Description: ${r.description}
 Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')}
 <!-- Truncated -->`).join('\n\n') : ''}`.trim())
 					}
-					if(mem.tool) tools.push(this.memoryManager.tools.read(mem.memory));
+					if(mem.tool) tools.push(...this.memoryManager.tools.read(mem.memory));
 				}
 			}
@@ -594,24 +596,6 @@ Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')}
 		return h;
 	}
 	/**
 	 * Compare the difference between embeddings (calculates the angle between two vectors)
 	 * @param {number[]} v1 First embedding / vector comparison
 	 * @param {number[]} v2 Second embedding / vector for comparison
 	 * @returns {number} Similarity values 0-1: 0 = unique, 1 = identical
 	 */
 	cosineSimilarity(v1: number[], v2: number[]): number {
 		if (v1.length !== v2.length) throw new Error('Vectors must be same length');
 		let dotProduct = 0, normA = 0, normB = 0;
 		for (let i = 0; i < v1.length; i++) {
 			dotProduct += v1[i] * v2[i];
 			normA += v1[i] * v1[i];
 			normB += v2[i] * v2[i];
 		}
 		const denominator = Math.sqrt(normA) * Math.sqrt(normB);
 		return denominator === 0 ? 0 : dotProduct / denominator;
 	}
 	/**
 	 * Chunk text into parts for AI digestion
 	 * @param {object | string} target Item that will be chunked (objects get converted)
@@ -1,111 +0,0 @@
 import {memoryStore} from '../memory.ts';
 import {Memory, MemoryStore} from './memory.ts';
 import {LLMRequest} from '../llm.ts';
 import {embedMemoryFields, journalDescription, stripHeader, touchHeader} from '../utils.ts';
 export const PENDING_HEADING = '## Pending';
 export const TODO_HEADING = '## Todo List';
 export async function docAgent(llm: any, node: Memory, memories: MemoryStore, options: LLMRequest, entry?: {request: {abort?: () => void} | null},): Promise<void> {
 	const store = memoryStore(memories);
 	if(!store.list.includes(node)) return;
 	const currentBody = stripHeader(node.content);
 	const journal = node.name.startsWith('Journal/');
 	const system = `You maintain a Markdown ${journal ? 'journal' : 'wiki file'}, produce the final document body
 - Incorporate the \`${PENDING_HEADING}\` into the wiki & remove it. These new facts trump conflicting information
 - Preserve existing structure, wording and context unless directly affected by \`${PENDING_HEADING}\` information
 - Only restructure if clearly unorganized, or there is duplicate, stale or misleading information. Do not restructure merely because you dont like it
 - Let the structure fit the document; do not force a template
 - Use headings, subheadings, lists, tables, code blocks and other formatting where useful
 - Create [[WikiLinks]] to existing pages or ghost nodes when the relationship is meaningful but does not exist yet
 - Remove duplication and obsolete information, preserve useful detail
 - Do not invent facts or make unnecessary changes
 - Never add edit commentary` + (journal ? `
 - This is a JOURNAL document, focus on creating a chronological report of events
 - Keep events chronological and concise. Do not reorganize events into
 - subject-based wiki sections. Preserve dates and useful temporal context.
 Rough Template:
 \`\`\`markdown
 # Journal
 ## ${TODO_HEADING}
 - [ ] ...
 ### YYYY-MM-DD
 - ...
 \`\`\`` : `
 - Use only sections that have content and make sense
 - Keep related facts together
 - historical context may be preserved
 - Use ## Related for meaningful links to other wiki pages.
 Rough Template:
 # Title
 ## ${TODO_HEADING}
 - [ ] ...
 ## Overview
 ...
 ## Heading
 ...
 ### Subheading
 ...
 ## Related
 - [[wikiLinks]]: relationship description
 `) + `
 Available pages for wikilinks:
 ${store.list.filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}`;
 	let update;
 	try {
 		for(let i = 0; i < 2 && !update?.content; i++) {
 			const request = llm.ask(currentBody, {
 				model: options.model,
 				temperature: 0.3,
 				schema: {
 					description: {
 						type: 'string',
 						description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint',
 						required: true,
 					},
 					content: {
 						type: 'string',
 						description: 'Rewritten document body in markdown, without the frontmatter block',
 						required: true,
 					},
 				},
 				system,
 			});
 			if(entry) entry.request = request;
 			update = await request;
 		}
 	} catch(err: any) {
 		if(err?.name === 'AbortError') return;
 		throw err;
 	} finally {
 		if(entry) entry.request = null;
 	}
 	if(!update?.content) return;
 	node.description = node.name.startsWith('Journal/')
 		? journalDescription(node.name)
 		: node.name !== 'People/User'
 			? update.description.replaceAll(/[\n:]/g, '')
 			: 'All information about the current user';
 	node.content = touchHeader(node, update.content);
 	await embedMemoryFields(node, llm);
 }
@@ -1,154 +0,0 @@
 import {MemoryCache, memoryStore} from '../memory.ts';
 import {LLMRequest} from '../llm.ts';
 import {Memory, MemoryStore} from './memory.ts';
 const ALIAS_MATCH_THRESHOLD = 0.55;
 export type FactBucket = {
 	subject: string;
 	facts: string[];
 }
 export type MemoryTask = {
 	subject: string;
 	task: string;
 	done: boolean;
 }
 export type FactAgentResult = {
 	buckets: FactBucket[];
 	journal: string;
 	tasks: MemoryTask[];
 }
 function dedupeFacts(facts: string[]): string[] {
 	const seen = new Map<string, string>();
 	for(const f of facts) {
 		const clean = f.trim();
 		if(clean) seen.set(clean.toLowerCase(), clean);
 	}
 	return [...seen.values()];
 }
 export function resolveSubject(subject: string, memories: Memory[] | MemoryCache, llm: any,): string {
 	function normalize(name: string): string {
 		return name.trim().toLowerCase().replace(/\s+/g, ' ');
 	}
 	const list = memories instanceof MemoryCache ? memories.memories : memories;
 	const trimmed = subject.trim();
 	const exact = list.find(m => m.name === trimmed);
 	if(exact) return exact.name;
 	const normalized = normalize(trimmed);
 	const caseInsensitive = list.find(m => normalize(m.name) === normalized);
 	if(caseInsensitive) return caseInsensitive.name;
 	const root = trimmed.split('/')[0];
 	const leaf = trimmed.split('/').slice(1).join('/') || trimmed;
 	const candidates = list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
 	if(!candidates.length) return trimmed;
 	const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
 	const probe = leaves.length > 1 ? leaves : [...leaves, ''];
 	const {max, similarities} = llm.fuzzyMatch(leaf, ...probe);
 	if(max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name;
 	return trimmed;
 }
 export async function factAgent(llm: any, conversation: string, memories: MemoryStore, options: LLMRequest,): Promise<FactAgentResult> {
 	const store = memoryStore(memories);
 	const ghosts = store.ghosts();
 	const response = await llm.ask(conversation, {
 		model: options.model,
 		temperature: 0.2,
 		system: `Turn this conversation into a persistent memory file by extracting information into organized bullet points
 Think of this like an Obsidian vault with a clear division of responsibility:
 - The JOURNAL is a chronological log. It answers "what happened, and when" and is the only place with a sense of time.
 - ENTITY DOSSIERS are a wiki pages. They answer "what is currently true about this subject", with no sense of time — only current state.
 - Never blur the two: a one-off event, conversation, or debugging session is a journal entry, not an entity, even if it's detailed.
 - Never extract this assistant's own tools, capabilities, or system behavior — that's not memory, it's spec.
 - Skip greetings, pleasantries, small talk and generic exchanges entirely.
 1. Journal Log
 - A chronological, skimmable log of what actually happened: high level discussions, decisions made (including ones reached jointly with the assistant), progress on projects, problems worked through
 - This is NOT a transcript, and it is NOT a step-by-step record, its a compressed day-to-day log of notable events & developments
 - One line per development is usually enough: what was worked on and the outcome, not the blow-by-blow of how
 - Skip anything that's a todo item (goes in Todo Tasks) or a durable fact about a subject (goes in Entity Dossiers)
 2. Todo Tasks
 - Extract concrete tasks the user says need to be done, should be done, or were completed
 - Return the task text and whether it is still todo or is done
 - A completed task should be marked done, not recreated as a new todo
 - Only extract actionable tasks, not general goals or observations, if none - omit returning a tasks array
 - Assign each task a HOME ENTITY subject using the same rules as Entity Dossiers below, or an empty string if it belongs in the journal: personal/life task (reach out to someone, pay a bill, etc.)
 3. Entity Dossiers
 - Detailed dossiers with all factual information regarding a subject
 - Only extract facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation — not assistant claims, guesses, or temporary/debugging details
 - Record the final/end state, not intermediate deltas
 - NEVER create a dossier for something I wouldn't find on a wiki: temporary info, debugging, guesses, conversation fragments (that's journal material)
 Entity Dossier Routing — every fact belongs to exactly one HOME ENTITY: the [pro]noun that OWNS the fact, not the initializer
 - All facts primarily about the user go under People/User
 - Path format is always Collection/Subject (People/Sarah, Projects/Oxide) — never a bare/rootless name
 - ALWAYS prefer an existing node (including ghosts) over creating a new one, even under a different alias — route the facts there, and record the unused name as a ghost node
 - Only use child paths (Collection/Subject/Aspect) when a clear child/parent relationship exists between entities
 - Use [[WikiLinks]] to express relationships between entities. Ghost links are supported so NEVER create a document just to hold a relationship
 Example Entity Naming Convention:
 - Projects/[Name]
 - People/[Name]
 - History/[Name]
 - Science/[Name]
 - [Subject]/[Name]
 - Class/[Name]/[Chapter]
 Available nodes:
 ${store.list.map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
 ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		schema: {
 			journal: {
 				type: 'string',
 				description: 'Short bullet point recap, omit if nothing notable happened',
 			},
 			tasks: {
 				type: 'array',
 				description: 'Concrete tasks mentioned or completed in the conversation, omit if none',
 				items: {type: 'object', items: {
 					subject: {type: 'string', description: 'Exact node name / new persistent entity path this task belongs to, or an empty string if this is a personal task with no entity of its own (those go in the journal)', required: true,},
 					task: {type: 'string', description: 'Concise actionable task', required: true,},
 					done: {type: 'boolean', description: 'Whether the task is completed', required: true,},
 				}},
 			},
 			buckets: {
 				type: 'array',
 				description: 'Groups of facts to remember; omit if none',
 				items: {type: 'object', items: {
 					subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true,},
 					facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'},},
 				}},
 			},
 		},
 	});
 	const buckets = new Map<string, string[]>();
 	for(const bucket of response.buckets ?? []) {
 		const subject = bucket.subject.trim();
 		const facts = buckets.get(subject) ?? [];
 		facts.push(...dedupeFacts(bucket.facts));
 		buckets.set(subject, facts);
 	}
 	return {
 		buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
 		journal: (response.journal ?? '').trim(),
 		tasks: response.tasks ?? [],
 	};
 }
@@ -1,4 +1,5 @@
-import {Memory, MemoryCache} from './memory.ts';
+import {MemoryCache} from './memory-state.ts';
 import type {Memory} from './memory.ts';
 export type MemoryNode = {
 	name: string;
@@ -13,14 +14,6 @@ export function extractLinks(content: string): string[] {
 	return [...new Set([...matches].map(m => m[1].trim()))];
 }
 /**
 * Incrementally patch the graph for a set of changed memories, instead of
 * re-scanning every document. Only the changed memories' own content is
 * re-parsed for links; affected targets have their backlinks patched.
 * Does NOT handle node deletion — full rebuildGraph() is still required
 * when a memory is removed, since that needs a backlink sweep across
 * everyone who might reference it.
 */
 export function patchGraph(mems: Memory[], nodes: MemoryNode[], changed: Memory[]): MemoryNode[] {
 	const nameSet = new Set(mems.map(m => m.name));
 	const byName = new Map(nodes.map(n => [n.name, n]));
@@ -1,79 +0,0 @@
 import {memoryStore} from '../memory.ts';
 import {Memory, MemoryStore} from './memory.ts';
 import {LLMRequest} from '../llm.ts';
 export type LibrarianAction =
 	| {type: 'merge'; source: string; target: string; instructions?: string}
 	| {type: 'split'; target: string; documents: {name: string; instructions?: string}[]}
 	| {type: 'move'; source: string; target: string}
 	| {type: 'delete'; target: string}
 	| {type: 'link'; source: string; target: string};
 export type LibrarianResult = {
 	actions: LibrarianAction[];
 };
 export async function librarianAgent(llm: any, node: Memory, memories: MemoryStore, options: LLMRequest, limit = 8,): Promise<LibrarianResult> {
 	if(node.name.startsWith('Journal/')) return {actions: []};
 	const store = memoryStore(memories);
 	const candidates = store.search(node.embedding, limit + 1)
 		.filter(result => result.name !== node.name && !result.name.startsWith('Journal/'))
 		.map(result => store.find(result.name))
 		.filter((memory): memory is Memory => !!memory);
 	if(!candidates.length) return {actions: []};
 	const documents = [node, ...candidates];
 	const response = await llm.ask('', {
 		model: options.model,
 		temperature: 0.2,
 		schema: {
 			actions: {
 				type: 'array',
 				description: 'Actions needed to improve the organization of the knowledge base. Omit if no changes are needed.',
 				items: {type: 'object', items: {
 					type: {type: 'string', description: 'One of: merge, split, move, delete, link', required: true,},
 					source: {type: 'string', description: 'Source document name. Required for merge, move, and link.',},
 					target: {type: 'string', description: 'Target document name. Required for merge, move, delete, and link.',},
 					instructions: {type: 'string', description: 'Specific instructions for performing the action.',},
 					documents: {type: 'array', description: 'Documents to create when splitting a document.', items: {type: 'object', items: {
 						name: {type: 'string', required: true},
 						instructions: {type: 'string'},
 					}}}},
 				},
 			},
 		},
 		system: `You are a librarian of a wiki. You keep it organized, coherent, and easy to navigate. You DONT rewrite pages but rather, recommend actions to organize the wiki
 Available actions:
 - merge: Two pages represent the same entity and should be merged together
 - split: One page contains significant information on two distinct entities and should become separate pages
 - move: A document belongs under a different name or path for consistancy
 - delete: A document is obsolete, accidental or contains nothing redeamable
 - link: Two distinct entities are meaningfully related and should reference each other
 Rules:
 - Only act on high confidence
 - Semantic similarity alone is NOT sufficient reason to merge two distinct entities
 - Prefer preserving existing entities and paths when possible
 - ALWAYS use & preserve [[wikilinks]] (including ghost nodes and journal entries) to represent relationships
 - A document should represent one persistent entity, not a temporal state, feature, bug, event, decision, or conversation
 - When merging, use the more common name as the destination and use ghost nodes to reference old aliases
 - When splitting, each resulting document must represent a distinct persistent entity
 - Return an empty actions array when the documents are already organized correctly
 Documents being reviewed:
 ${documents.map(memory => `### ${memory.name}
 Description: ${memory.description}
 Links: ${[...memory.links, ...memory.backlinks].join(', ') || 'none'}
 \`\`\`markdown
 ${memory.content}
 \`\`\``).join('\n\n')}`,
 	});
 	return {
 		actions: response.actions ?? [],
 	};
 }
@@ -1,7 +1,7 @@
-import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts';
+import {MemoryNode, patchGraph, rebuildGraph} from './graph.ts';
-import {KDTree} from './memory/kd-tree.ts';
+import {KDTree} from './kd-tree.ts';
-import {Memory, MemoryRef, MemoryStore} from './memory/memory.ts';
+import type {Memory, MemoryRef, MemoryStore} from './memory.ts';
-import {cosineDistance, embedMemoryFields} from './utils.ts';
+import {cosineDistance, embedMemoryFields} from '../utils.ts';
 const TREE_TOMBSTONE_LIMIT = 0.25;
@@ -45,6 +45,7 @@ export function memoryStore(memories: MemoryStore): {
 		forget: name => {
 			const idx = memories.findIndex(m => m.name === name);
 			if(idx === -1) return false;
 			memories.splice(idx, 1);
 			return true;
 		},
@@ -54,14 +55,7 @@ export function memoryStore(memories: MemoryStore): {
 			if(!missing.length) return 0;
 			await Promise.all(missing.map(async node => {
-				const body = node.content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
+				await embedMemoryFields(node, llm);
 				const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name);
 				const [descE] = await llm.embedding(node.description || '');
 				const bodyChunks = body ? await llm.embedding(body) : [];
 				if(titleE) node.titleEmbedding = titleE.embedding;
 				if(descE) node.embedding = descE.embedding;
 				node.bodyEmbeddings = bodyChunks.map((c: any) => c.embedding).filter(Boolean);
 			}));
 			return missing.length;
@@ -94,6 +88,7 @@ export class MemoryCache {
 		for(const [name, emb] of [...this.indexed]) {
 			const mem = this.memories.find(m => m.name === name);
 			if(!mem || !current.has(name) || mem.embedding !== emb) {
 				this.tree.remove(p => p.name === name);
 				this.indexed.delete(name);
@@ -102,21 +97,36 @@ export class MemoryCache {
 		for(const mem of this.memories) {
 			if(!mem.embedding?.length || this.indexed.has(mem.name)) continue;
-			if(this.tree.dims === 0) this.tree = new KDTree<MemoryRef>(mem.embedding.length, 'cosine');
+
 			if(this.tree.dims === 0) {
 				this.tree = new KDTree<MemoryRef>(mem.embedding.length, 'cosine');
 			}
 			if(mem.embedding.length !== this.tree.dims) continue;
 			this.tree.insert({
 				vector: mem.embedding,
-				payload: {name: mem.name, description: mem.description},
+				payload: {
 					name: mem.name,
 					description: mem.description,
 				},
 			});
 			this.indexed.set(mem.name, mem.embedding);
 		}
-		if(this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance();
+		if(this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) {
 			this.tree.rebalance();
 		}
 	}
 	search(query: number[], limit: number): MemoryRef[] {
 		if(!this.tree || this.tree.dims === 0) return [];
-		return this.tree.knn(query, limit).map(r => ({...r.point.payload, distance: r.distance}));
+
 		return this.tree.knn(query, limit).map(r => ({
 			...r.point.payload,
 			distance: r.distance,
 		}));
 	}
 	add(memory: Memory): void {
@@ -126,14 +136,17 @@ export class MemoryCache {
 	update(memory: Memory): void {
 		const existing = this.find(memory.name);
 		if(existing) Object.assign(existing, memory);
 		else this.memories.push(memory);
 		this.rebuild([existing ?? memory]);
 	}
 	remove(name: string): boolean {
 		const idx = this.memories.findIndex(m => m.name === name);
 		if(idx === -1) return false;
 		this.memories.splice(idx, 1);
 		this.rebuild();
 		return true;
@@ -147,6 +160,7 @@ export class MemoryCache {
 		this.nodes = changed?.length && this.nodes.length
 			? patchGraph(this.memories, this.nodes, changed)
 			: rebuildGraph(this.memories);
 		this.syncTree();
 		return this.nodes;
 	}
@@ -158,8 +172,10 @@ export class MemoryCache {
 	async backfillEmbeddings(llm: any): Promise<number> {
 		const missing = this.memories.filter(m => !m.embedding?.length);
 		if(!missing.length) return 0;
 		await Promise.all(missing.map(node => embedMemoryFields(node, llm)));
 		this.commit(missing);
 		return missing.length;
 	}
 }
@@ -1,4 +1,12 @@
-import {MemoryCache} from '../memory.ts';
+import {AiTool} from '../tools.ts';
 import type {LLMMessage, LLMRequest} from '../llm.ts';
 import {MemoryCache, memoryStore} from './memory-state.ts';
 import {cosineDistance, embedMemoryFields, stripHeader, updateMemory} from '../utils.ts';
 const FACT_SIMILARITY_THRESHOLD = 0.62;
 const DUPLICATE_THRESHOLD = 0.68;
 const PROTECTED_MEMORIES = ['People/User'];
 const COLLECTION_WORDS = ['project', 'projects', 'people', 'person', 'managed', 'guides', 'guide', 'research', 'class', 'classes'];
 export type Memory = {
 	name: string;
@@ -26,3 +34,315 @@ export type MemoryOptions = {
 }
 export type MemoryStore = Memory[] | MemoryCache;
 /** Create an empty memory shell. */
 function emptyNode(name: string, description = ''): Memory {
 	return {name, description, content: `# ${name.split('/').pop()}\n`, embedding: [], links: [], backlinks: []};
 }
 function renderNode(node: Memory): string {
 	return `### ${node.name}
 Description: ${node.description}
 Links: ${[...node.links, ...node.backlinks].join(', ') || 'none'}
 \`\`\`markdown
 ${node.content}
 \`\`\``;
 }
 function factSimilarity(a: Memory, b: Memory): number {
 	return !a.bodyEmbeddings?.length || !b.bodyEmbeddings?.length ? 0 : Math.max(...a.bodyEmbeddings.flatMap(av => b.bodyEmbeddings!.map(bv => 1 - cosineDistance(av, bv))));
 }
 function words(text: string): string[] {
 	return [...new Set(text.toLowerCase().replace(/[[\]()/_-]/g, ' ').replace(/[^a-z0-9\s]/g, '').split(/\s+/).filter(w => w && !COLLECTION_WORDS.includes(w)))];
 }
 function jaccard(a: string[], b: string[]): number {
 	const bs = new Set(b), hit = a.filter(x => bs.has(x)).length, total = new Set([...a, ...b]).size;
 	return total ? hit / total : 0;
 }
 function duplicateScore(a: Memory, b: Memory): number {
 	const name = Math.max(
 		jaccard(words(a.name), words(b.name)),
 		jaccard(words(a.name.split('/').pop() || a.name), words(b.name.split('/').pop() || b.name)),
 	);
 	const desc = jaccard(words(a.description), words(b.description));
 	const body = factSimilarity(a, b);
 	const emb = a.embedding?.length && b.embedding?.length && a.embedding.length === b.embedding.length ? 1 - cosineDistance(a.embedding, b.embedding) : 0;
 	return Math.max(body, name * 0.9 + desc * 0.06 + emb * 0.04, emb * 0.55 + name * 0.35 + desc * 0.1);
 }
 function homeScore(node: Memory): number {
 	return (PROTECTED_MEMORIES.includes(node.name) ? 1e9 : 0)
 		+ (node.name.includes('/') ? 4 : 0)
 		+ (node.description && node.description !== 'Persistent memory document' ? 1 : 0)
 		+ Math.min(stripHeader(node.content).length / 1000, 5);
 }
 function pickMerge(a: Memory, b: Memory, touched: Set<string>): [drop: Memory, home: Memory] {
 	const as = homeScore(a), bs = homeScore(b);
 	if(touched.has(a.name) && !touched.has(b.name)) return as > bs + 2 ? [b, a] : [a, b];
 	if(touched.has(b.name) && !touched.has(a.name)) return bs > as + 2 ? [a, b] : [b, a];
 	return as <= bs ? [a, b] : [b, a];
 }
 /** Build memory tools and memory index text. */
 export function memoryTools(llm: any, memories: MemoryStore): {tools: AiTool[]; list: string} {
 	const store = memoryStore(memories);
 	const names = new Map<string, string>();
 	for(const node of store.list)
 		if(!names.has(node.name)) names.set(node.name, `${node.name} - ${node.description}`);
 	for(const name of store.ghosts())
 		if(!names.has(name)) names.set(name, `${name} - ghost node`);
 	return {
 		list: [...names.values()].join('\n'),
 		tools: [
 			{
 				name: 'memory_search',
 				description: 'Semantically search memories for most relevant',
 				args: {
 					query: {type: 'string', description: 'Search query', required: true},
 					limit: {type: 'number', description: 'Maximum results, default 5', default: 5},
 				},
 				fn: async ({query, limit = 5}) => {
 					if(!query?.trim()) return 'Search query is required.';
 					const [chunk] = await llm.embedding(query, {maxTokens: 8000, overlapTokens: 0});
 					if(!chunk?.embedding) return 'Failed to create embedding from query';
 					const results = store.search(chunk.embedding, limit).map(ref => store.find(ref.name)).filter((node): node is Memory => !!node);
 					return results.length ? results.map(renderNode).join('\n\n---\n\n') : 'No relevant memories found.';
 				},
 			},
 			{
 				name: 'memory_read',
 				description: 'Read an entire memory document by name',
 				args: {name: {type: 'string', description: 'Exact document name', required: true}},
 				fn: async ({name}) => {
 					const node = store.find(name);
 					return node ? renderNode(node) : store.ghosts().includes(name) ? `"${name}" is a ghost node with no document of its own.` : `Not found: "${name}".`;
 				},
 			},
 			{
 				name: 'memory_delete',
 				description: 'Delete a duplicate or merged memory',
 				args: {name: {type: 'string', description: 'Exact document name', required: true}},
 				fn: async ({name}) => {
 					store.forget(name);
 					return `Removed: ${name}`;
 				},
 			},
 			{
 				name: 'memory_write',
 				description: 'Create or replace a memory document.',
 				args: {
 					name: {type: 'string', description: 'Document name following the entity naming convention.', required: true},
 					description: {type: 'string', description: 'One factual sentence describing the entire document subject', required: true},
 					content: {type: 'string', description: 'Complete Markdown document body, including the # title', required: true},
 				},
 				fn: async (args: any) => {
 					const name = String(args.name || '').trim();
 					if(!name) return 'A document name is required.';
 					const description = String(args.description || '').trim();
 					if(!description) return 'A document description is required.';
 					const content = String(args.content || '').trim();
 					if(!content) return 'Document content is required.';
 					let node = store.find(name);
 					if(!node) {
 						node = emptyNode(name, description);
 						if(store.cache) store.cache.add(node);
 						else store.list.push(node);
 					}
 					node.description = name === 'People/User' ? 'All information about the current user' : description.replace(/\s+/g, ' ').trim();
 					node.content = updateMemory(node, content);
 					await embedMemoryFields(node, llm);
 					store.cache?.commit([node]);
 					return `Updated ${name}`;
 				},
 			},
 		],
 	};
 }
 export class MemoryManager {
 	private memorized = new WeakMap<LLMMessage[], LLMMessage>();
 	constructor(private llm: any) {}
 	static normalize(memory?: Memory[] | MemoryCache | MemoryOptions): MemoryOptions | null {
 		if(!memory) return null;
 		if(Array.isArray(memory) || memory instanceof MemoryCache) return {memory, inject: true, tool: false, update: false};
 		if(typeof memory === 'object' && 'memory' in memory) return {inject: true, tool: false, update: false, ...memory};
 		return null;
 	}
 	private memorySystem(list: string): string {
 		return `You maintain notes written in markdown used for memories from recent conversations using your tools.
 Only preserve durable information worth remembering established by the USER.
 Do not store assistant guesses, speculation, suggestions, commentary, temporary state, or details that are not worth remembering.
 ## Rules
 - ALWAYS READ a target memory before changing it, \`memory_write\` does a full replace, it DOES NOT append!
 - Memories should contain the final state, not deltas
 - New conversational context is authoritative when it contracts existing information; reconcile it
 - Only remove information when stale, contradicted or duplicated; always preserve existing information, formatting and keep related information together
 - Only merge memories when two or more nodes are clearly about the same thing; only split a memory when it is clearly about two distinct subjects
 - Use [[WikiLinks]] liberally to record aliases and relationships between entities, even ones without pages yet (ghost nodes)
 - Use headings, subheadings, lists, tables and other markdown formatting to make documents clean
 - Maintain a \`## Todo List\` of checkboxes AS THE FIRST SUBHEADING when an entity has tasks
 	- Only create todo items for USER tasks, not AI work
 	- Only store each in one place, no duplicates
 	- Use  \`People/User\` for personal tasks or as a fallback
 ## Naming
 - Every fact should be grouped with the owning entity
 - Always follow the naming convention \`Collection/(Pro)Noun\`
 - Facts about the user belong under People/User
 - Reuse existing memories when they are clearly the same entity including aliases and ghost references.
 - Only create deeper paths when there is a real parent/child entity relationship: \`School/Class/Chapter\`
 Valid Examples:
 - People/User
 - People/John Smith
 - Projects/Momentum
 - Projects/Momentum/Marketing
 - Research/Object Recognition
 - Guides/HAM Radio SOP
 ## Workflow
 1. Create groups of durable information and todos based on the owning entity & naming rules above
 2. For each group:
  1. Read the existing memory(s)
  2. Merge the information & todos based on the rules above
  3. Write the entire patched document
 Available memories:
 ${list || 'No memory documents exist yet.'}`;
 	}
 	private touchedNames(history: LLMMessage[]): string[] {
 		return [...new Set(history
 			.filter((h: any) => h.role === 'tool' && h.name === 'memory_write' && !h.error)
 			.map((h: any) => String(h.args?.name || h.content?.match(/^Updated (.+)$/)?.[1] || '').trim())
 			.filter(Boolean))];
 	}
 	private async backfillEmbeddings(store: ReturnType<typeof memoryStore>): Promise<void> {
 		const missing = store.list.filter(m => !m.embedding?.length || !m.titleEmbedding?.length || !m.bodyEmbeddings?.length);
 		await Promise.all(missing.map(m => embedMemoryFields(m, this.llm)));
 		store.cache?.commit(missing);
 	}
 	private closestDuplicate(node: Memory, store: ReturnType<typeof memoryStore>): Memory | null {
 		return store.list
 			.filter(m => m.name !== node.name && !m.name.startsWith('Journal/') && !node.name.startsWith('Journal/'))
 			.map(m => ({node: m, score: duplicateScore(node, m)}))
 			.filter(x => x.score >= DUPLICATE_THRESHOLD || factSimilarity(node, x.node) >= FACT_SIMILARITY_THRESHOLD)
 			.sort((a, b) => b.score - a.score)[0]?.node || null;
 	}
 	private async rehomeDeleted(drop: Memory, home: Memory, memories: MemoryStore, options: LLMRequest): Promise<void> {
 		const store = memoryStore(memories);
 		const backup = structuredClone(drop);
 		store.forget(drop.name);
 		try {
 			const memory = memoryTools(this.llm, memories);
 			await this.llm.ask(`A duplicate memory document was removed automatically.
 Deleted document:
 ${renderNode(backup)}
 Closest surviving home:
 ${renderNode(home)}
 Reinsert every durable unique fact, useful relationship, alias, and user todo from the deleted document into the best remaining memory document.
 Usually this should be "${home.name}", but use another existing memory if it is a better home.
 Read before writing. Write full replacement documents only.
 Do NOT recreate "${backup.name}" unless the deletion was wrong and it is clearly a distinct persistent entity.`, {
 				model: options.memoryModel || options.model,
 				temperature: 0.2,
 				maxTokens: options.maxTokens,
 				tools: memory.tools,
 				history: [],
 				system: this.memorySystem(memory.list),
 			});
 		} catch(err) {
 			if(!store.find(backup.name)) store.cache ? store.cache.add(backup) : store.list.push(backup);
 			throw err;
 		} finally {
 			store.cache?.commit(store.list);
 		}
 	}
 	private async reconcileSimilar(history: LLMMessage[], memories: MemoryStore, options: LLMRequest): Promise<void> {
 		const store = memoryStore(memories);
 		const touched = new Set(this.touchedNames(history));
 		const targets = store.list.filter(m => touched.has(m.name) || [...touched].some(t => duplicateScore(m, store.find(t) || m) >= DUPLICATE_THRESHOLD));
 		const deleted = new Set<string>();
 		if(!targets.length) return;
 		await this.backfillEmbeddings(store);
 		for(const node of targets) {
 			if(!store.find(node.name) || deleted.has(node.name) || PROTECTED_MEMORIES.includes(node.name)) continue;
 			const closest = this.closestDuplicate(node, store);
 			if(!closest) continue;
 			const [drop, home] = pickMerge(node, closest, touched);
 			if(deleted.has(drop.name) || PROTECTED_MEMORIES.includes(drop.name)) continue;
 			deleted.add(drop.name);
 			await this.rehomeDeleted(drop, home, memories, options);
 			await this.backfillEmbeddings(store);
 		}
 	}
 	async recollect(query: string, memory: MemoryStore, limit = 15): Promise<Memory[]> {
 		const store = memoryStore(memory);
 		if(!store.list.length || !query?.trim()) return [];
 		const [chunk] = await this.llm.embedding(query, {maxTokens: 8000, overlapTokens: 0});
 		return !chunk?.embedding ? [] : store.search(chunk.embedding, limit).map(ref => store.find(ref.name)).filter((m: Memory | undefined): m is Memory => !!m);
 	}
 	get tools(): {read: (memory: MemoryStore) => AiTool[]} {
 		return {read: (memory: MemoryStore) => memoryTools(this.llm, memory).tools};
 	}
 	async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest = {},): Promise<Memory[]> {
 		const store = memoryStore(memories);
 		const previous = this.memorized.get(history);
 		let start = 0;
 		if(previous) {
 			const index = history.indexOf(previous);
 			if(index >= 0) start = index + 1;
 		}
 		const turns = history.slice(start).filter((h: any) => h.role === 'user' || h.role === 'assistant');
 		const conversation = turns.map((h: any) => `[${h.role}]: ${h.content}`).join('\n\n').trim();
 		if(!conversation) return store.list;
 		const memory = memoryTools(this.llm, memories);
 		const memoryHistory: LLMMessage[] = [];
 		await this.llm.ask(conversation, {
 			model: options.memoryModel || options.model,
 			temperature: 0.2,
 			maxTokens: options.maxTokens,
 			tools: memory.tools,
 			history: memoryHistory,
 			system: this.memorySystem(memory.list),
 		});
 		await this.reconcileSimilar(memoryHistory, memories, options);
 		const lastTurn = turns.at(-1);
 		if(lastTurn) this.memorized.set(history, lastTurn);
 		return store.list;
 	}
 }
@@ -30,7 +30,7 @@ export function euclideanDistance(a: number[], b: number[]): number {
 	return Math.sqrt(sum);
 }
-function getWeekStart(date: Date = new Date()): string {
+export function getWeekStart(date: Date = new Date()): string {
 	const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
 	const day = d.getUTCDay();
 	const diff = day === 0 ? -6 : 1 - day;
@@ -72,7 +72,7 @@ export function stripHeader(content: string): string {
 	return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
 }
-export function touchHeader(node: Memory, body: string): string {
+export function updateMemory(node: Memory, body: string): string {
 	const {fm} = parseFrontmatter(node.content);
 	fm.set('name', node.name);
 	fm.set('description', (node.name.startsWith('Journal/') ? journalDescription(node.name) : node.description)