ai-utils/src/memory.ts

import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts';
import {LLMRequest, LLMMessage} from './llm.ts';
import {AiTool} from './tools.ts';
import {KDTree} from './kd-tree.ts';

const FACT_SIMILARITY_THRESHOLD = 0.62;
const PENDING_HEADING = '## Pending';
const TODO_HEADING = '## Todo list';
const TREE_TOMBSTONE_LIMIT = 0.25;
const ALIAS_MATCH_THRESHOLD = 0.55;

export type Memory = {
	name: string;
	description: string;
	content: string;
	embedding: number[];
	titleEmbedding?: number[];
	bodyEmbeddings?: number[][];
	links: string[];
	backlinks: string[];
}

type MemoryRef = {
	name: string;
	description: string;
	distance?: number;
}

type FactBucket = {
	subject: string;
	facts: string[];
}

type MemoryTask = {
	/** Exact node name / new persistent entity path this task belongs to, or '' for a personal task with no entity (goes to the journal) */
	subject: string;
	task: string;
	done: boolean;
}

type FactAgentResult = {
	buckets: FactBucket[];
	journal: string;
	tasks: MemoryTask[];
}

function dedupeFacts(facts: string[]): string[] {
	const seen = new Map<string, string>();
	for(const f of facts) {
		const clean = f.trim();
		if(clean) seen.set(clean.toLowerCase(), clean);
	}
	return [...seen.values()];
}

function cosineDistance(a: number[], b: number[]): number {
	let dot = 0, normA = 0, normB = 0;
	for(let i = 0; i < a.length; i++) {
		dot += a[i] * b[i];
		normA += a[i] * a[i];
		normB += b[i] * b[i];
	}
	const denom = Math.sqrt(normA) * Math.sqrt(normB);
	return denom === 0 ? 1 : 1 - dot / denom;
}

function cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] {
	return memories
		.filter(m => m.embedding?.length)
		.map(m => ({name: m.name, description: m.description, distance: cosineDistance(query, m.embedding)}))
		.sort((a, b) => a.distance - b.distance)
		.slice(0, limit);
}

async function embedMemoryFields(node: Memory, llm: any): Promise<void> {
	const body = stripHeader(node.content);
	const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name);
	const [descE] = await llm.embedding(node.description || '');
	const bodyChunks = body ? await llm.embedding(body) : [];
	if(titleE) node.titleEmbedding = titleE.embedding;
	if(descE) node.embedding = descE.embedding;
	node.bodyEmbeddings = bodyChunks.map((c: any) => c.embedding).filter(Boolean);
}

export function stripHeader(content: string): string {
	return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
}

/** True if a task has no persistent entity of its own and belongs in the journal instead. */
function isPersonalTask(t: MemoryTask): boolean {
	const s = (t.subject ?? '').trim().toLowerCase();
	return !s || s === 'journal' || s.startsWith('journal/');
}

export class MemoryCache {
	private tree!: KDTree<MemoryRef>;
	private indexed = new Map<string, number[]>();
	public memories: Memory[];
	public nodes: MemoryNode[] = [];

	get length() { return this.memories.length; }

	constructor(memories: Memory[]) {
		this.memories = memories;
		this.tree = new KDTree<MemoryRef>(0);
		this.rebuild();
	}

	private syncTree(): void {
		const current = new Set(this.memories.map(m => m.name));

		for(const [name, emb] of [...this.indexed]) {
			const mem = this.memories.find(m => m.name === name);
			if(!mem || !current.has(name) || mem.embedding !== emb) {
				this.tree.remove(p => p.name === name);
				this.indexed.delete(name);
			}
		}

		for(const mem of this.memories) {
			if(!mem.embedding?.length || this.indexed.has(mem.name)) continue;
			if(this.tree.dims === 0) this.tree = new KDTree<MemoryRef>(mem.embedding.length, 'cosine');
			if(mem.embedding.length !== this.tree.dims) continue; // guard against embedding model/dim drift
			this.tree.insert({vector: mem.embedding, payload: {name: mem.name, description: mem.description}});
			this.indexed.set(mem.name, mem.embedding);
		}

		if(this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance();
	}

	search(query: number[], limit: number): MemoryRef[] {
		if(!this.tree || this.tree.dims === 0) return [];
		return this.tree.knn(query, limit).map(r => ({...r.point.payload, distance: r.distance}));
	}

	add(memory: Memory): void {
		this.memories.push(memory);
		this.rebuild([memory]);
	}

	update(memory: Memory): void {
		const existing = this.memories.find(m => m.name === memory.name);
		if(existing) Object.assign(existing, memory);
		else this.memories.push(memory);
		this.rebuild([existing ?? memory]);
	}

	remove(name: string): void {
		const idx = this.memories.findIndex(m => m.name === name);
		if(idx !== -1) {
			this.memories.splice(idx, 1);
			this.rebuild();
		}
	}

	rebuild(changed?: Memory[]): void {
		this.nodes = (changed?.length && this.nodes.length)
			? patchGraph(this.memories, this.nodes, changed)
			: rebuildGraph(this.memories);
		this.syncTree();
	}
}

class MemoryAccessor {
	readonly list: Memory[];
	private readonly cache: MemoryCache | null;

	constructor(memories: Memory[] | MemoryCache) {
		this.cache = memories instanceof MemoryCache ? memories : null;
		this.list = this.cache ? this.cache.memories : <Memory[]>memories;
	}

	find(name: string): Memory | undefined {
		return this.list.find(m => m.name === name);
	}

	commit(changed?: Memory[]): MemoryNode[] {
		if(this.cache) {
			this.cache.rebuild(changed);
			return this.cache.nodes;
		}
		return rebuildGraph(this.list);
	}

	ghosts(): string[] {
		const nodes = this.cache ? this.cache.nodes : rebuildGraph(this.list);
		return nodes.filter(n => n.missing).map(n => n.name);
	}

	search(vector: number[], limit: number): MemoryRef[] {
		return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit);
	}

	forget(name: string): boolean {
		const idx = this.list.findIndex(m => m.name === name);
		if(idx === -1) return false;
		this.list.splice(idx, 1);
		this.commit();
		return true;
	}

	async backfillEmbeddings(llm: any): Promise<number> {
		const missing = this.list.filter(m => !m.embedding?.length);
		if(!missing.length) return 0;
		await Promise.all(missing.map(node => embedMemoryFields(node, llm)));
		this.commit();
		return missing.length;
	}
}

export type MemoryOptions = {
	/** Memory object */
	memory: Memory[] | MemoryCache;
	/** Inject N memories into the system prompt */
	inject?: boolean;
	/** expose recall tool to LLM */
	tool?: boolean;
	/** Update memory on compression */
	update?: boolean;
	/** Max context size of memories to inject to each call (removed immediately after use) */
	maxTokens?: number;
}

export class MemoryManager {
	private mergeLock: Promise<any> = Promise.resolve();
	private queues = new Map<string, {
		dirty: boolean,
		request: {abort?: () => void} | null,
		task: Promise<void>,
	}>();
	private recentlyTouched = new Map<string, number>();

	tools = {
		forget: (memories: Memory[] | MemoryCache): AiTool => ({
			name: 'memory_forget',
			description: 'Permanently delete a memory document and clean up all references to it',
			args: {
				name: {type: 'string', description: 'Exact memory name to forget', required: true}
			},
			fn: (args: any) => {
				const result = this.forget(args.name, memories);
				return result ? `Forgotten: ${args.name}` : `Not found: ${args.name}`;
			},
		}),

		read: (memories: Memory[] | MemoryCache): AiTool => ({
			name: 'memory_recall',
			description: 'Read the full content of a memory document',
			args: {
				name: {type: 'string', description: 'Exact memory name', required: true}
			},
			fn: (args: any) => {
				const mem = new MemoryAccessor(memories).find(args.name);
				if(!mem) return 'Document not found';
				this.touch(mem.name);
				return mem.content;
			},
		}),

		search: (memories: Memory[] | MemoryCache): AiTool => ({
			name: 'memory_search',
			description: 'Use embeddings to find the MOST relevant memories, even if NOT relevant',
			args: {
				query: {type: 'string', description: 'What to look for in the memories', required: true},
				limit: {type: 'number', description: 'Number of memories to return', default: 1},
			},
			fn: async ({query, limit}) => {
				const mem = await this.recollect(query, memories, limit);
				return mem.map(m => `Memory: ${m.name}
Description: ${m.description}
Links: ${[...m.links, ...m.backlinks].join(', ')}
\`\`\`
${m.content}
\`\`\``).join('\n\n');
			},
		}),
	};

	constructor(private llm: any) {}

	static normalize(m?: Memory[] | MemoryCache | MemoryOptions) {
		if(!m) return null;
		const raw = m instanceof MemoryCache || Array.isArray(m);
		return raw ? {memory: <Memory[] | MemoryCache>m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m};
	}

	private stage(node: Memory, block: string): void {
		if(!node.content) {
			const title = node.name.split('/').pop() ?? node.name;
			node.content = this.touchHeader(node, `# ${title}\n`);
		}
		const body = stripHeader(node.content);
		const idx = body.indexOf(PENDING_HEADING);
		const newBody = idx === -1
			? `${body.trimEnd()}\n\n${PENDING_HEADING}\n${block}\n`
			: `${body.slice(0, idx + PENDING_HEADING.length)}\n${block}${body.slice(idx + PENDING_HEADING.length)}`;
		node.content = this.touchHeader(node, newBody);
	}

	private resolveSubject(subject: string, store: MemoryAccessor): string {
		function normalize(name: string): string {
			return name.trim().toLowerCase().replace(/\s+/g, ' ');
		}

		const trimmed = subject.trim();
		const exact = store.find(trimmed);
		if(exact) return exact.name;

		const normalized = normalize(trimmed);
		const caseInsensitive = store.list.find(m => normalize(m.name) === normalized);
		if(caseInsensitive) return caseInsensitive.name;

		const root = trimmed.split('/')[0];
		const leaf = trimmed.split('/').slice(1).join('/') || trimmed;
		const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
		if(!candidates.length) return trimmed;

		const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
		const probe = leaves.length > 1 ? leaves : [...leaves, ''];
		const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe);
		if(max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name;

		return trimmed;
	}

	private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest): Promise<FactAgentResult> {
		const ghosts = store.ghosts();

		const response = await this.llm.ask(conversation, {
			model: options.model,
			temperature: 0.2,
			system: `Turn this conversation into a persistent memory file by extracting information into organized bullet points

Think of this like an Obsidian vault with a clear division of responsibility:
- The JOURNAL is a timeline. It answers "what happened, and when" and is the only place with a sense of time.
- ENTITY DOSSIERS are a wiki. They answer "what is currently true about this subject", with no sense of time — only current state.
- Never blur the two: a one-off event, conversation, or debugging session is a journal entry, not an entity, even if it's detailed.

1. Journal Log
- A chronological, skimmable log of what actually happened: real discussions, decisions made, progress on projects, problems worked through
- This is NOT a transcript, and it is NOT a step-by-step record, its a compressed log of notable events & developments
- One line per development is usually enough: what was worked on and the outcome, not the blow-by-blow of how
- Skip small talk and trivial exchanges entirely. Skip anything that's purely a todo item (goes in Todo Tasks) or a durable fact about a subject (goes in Entity Dossiers)

2. Todo Tasks
- Extract concrete tasks the user says need to be done, should be done, or were completed
- Return the task text and whether it is still todo or is done
- A completed task should be marked done, not recreated as a new todo
- Only extract actionable tasks, not general goals or observations
- Assign each task a subject:
    - If the task belongs to a persistent entity (a project, a class, etc.), use that entity's exact node name, or a new entity path if it doesn't exist yet
    - If it's a personal/life task with no entity of its own (reach out to someone, reply to an email, pay a bill, etc.), leave subject as an empty string — it belongs in the journal, not a new document

3. Entity Dossiers
- Detailed dossiers with all information regarding a subject
- Record the final/end state, not intermediate changes
- Ignore assistant claims, guesses, greetings, or temporary details
- NEVER create a document for something that's only meaningful as a point in time — a single conversation, a one-off decision, a debugging session, a date. That's a journal entry, not an entity
- identify its HOME ENTITY:
    - The HOME ENTITY name should always be a [abstract|pro]noun
    - The grammatical subject/owner of the fact is the strongest clue
    - Always preference an existing entity over creating a new one
    - New child entities are appropriate only when they are themselves distinct persistent entities
    - A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment
    - Put project facts under the project they belong to, person facts under the person, etc

Example Entity Naming Convention:
- Projects/[Name]
- People/[Name]
- History/[Name]
- Science/[Name]
- [Subject]/[Name]
- Class/[Name]/[Chapter]

Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships
Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent

Available nodes:
${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
			schema: {
				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
				tasks: {
					type: 'array', description: 'Concrete tasks mentioned or completed in the conversation.', required: false, items: {
						type: 'object', items: {
							subject: {type: 'string', description: 'Exact node name / new persistent entity path this task belongs to, or an empty string if this is a personal task with no entity of its own (those go in the journal)', required: true},
							task: {type: 'string', description: 'Concise actionable task', required: true},
							done: {type: 'boolean', description: 'Whether the task is completed', required: true},
						},
					}
				},
				buckets: {
					type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
						type: 'object', items: {
							subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true},
							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
						},
					},
				},
			},
		});

		const buckets = new Map<string, string[]>();
		for(const bucket of response.buckets ?? []) {
			const subject = bucket.subject.trim();
			const facts = buckets.get(subject) ?? [];
			facts.push(...dedupeFacts(bucket.facts));
			buckets.set(subject, facts);
		}

		return {
			buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
			journal: (response.journal ?? '').trim(),
			tasks: response.tasks ?? [],
		};
	}

	private getWeekStart(date: Date = new Date()): string {
		const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
		const day = d.getUTCDay();
		const diff = day === 0 ? -6 : 1 - day;
		d.setUTCDate(d.getUTCDate() + diff);
		return d.toISOString().slice(0, 10);
	}

	private journalDescription(journalName?: string): string {
		const start = journalName?.split('/').pop() || this.getWeekStart();
		const d = new Date(`${start}T00:00:00Z`);
		d.setUTCDate(d.getUTCDate() + 6);
		const end = d.toISOString().slice(0, 10);
		return `Log from ${start} - ${end}`;
	}

	private getIncompleteTodos(content: string): string[] {
		const body = stripHeader(content);
		const match = body.match(/## Todo list\n([\s\S]*?)(?=\n## |$)/i);
		if(!match) return [];
		return match[1].split('\n')
			.map(line => line.match(/^\s*-\s*\[([ xX])\]\s+(.+?)\s*$/))
			.filter((m): m is RegExpMatchArray => !!m && m[1].toLowerCase() !== 'x')
			.map(m => m[2].trim());
	}

	private listNodes(memories: Memory[]): MemoryRef[] {
		return memories.map(m => ({name: m.name, description: m.description}));
	}

	private async mergeAgent(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory | null> {
		function factSimilarity(a: Memory, b: Memory): number {
			if(!a.bodyEmbeddings?.length || !b.bodyEmbeddings?.length) return 0;
			let best = 0;
			for(const av of a.bodyEmbeddings) {
				for(const bv of b.bodyEmbeddings) best = Math.max(best, 1 - cosineDistance(av, bv));
			}
			return best;
		}

		if(!node.embedding?.length || node.name.startsWith('Journal/')) return null;
		const store = new MemoryAccessor(memories);
		const candidates = store.list
			.filter(m => m.name !== node.name && !m.name.startsWith('Journal/'))
			.filter(m => factSimilarity(node, m) >= FACT_SIMILARITY_THRESHOLD);

		if(!candidates.length) return null;
		const closest = candidates.sort((a, b) => factSimilarity(node, b) - factSimilarity(node, a))[0];
		const result = await this.llm.ask('', {
			model: options.model,
			temperature: 0.3,
			schema: {
				aContent: {type: 'string', description: 'Updated document A body in markdown, without frontmatter.', required: true},
				bContent: {type: 'string', description: 'Updated document B body in markdown, without frontmatter.', required: true},
			},
			system: `Maintain these two persistent knowledge-base documents like a wiki.

Do NOT merge, rename, or delete either document. Both represent entities that should remain independently addressable.

The documents were selected because their facts may overlap. Your job is to reconcile duplicated information and connect the documents:
- Decide which document is the HOME for each duplicated fact.
- Keep the authoritative copy in that home document.
- In the other document, replace the information with a short preamble and [[WikiLink]] to the home entity explaining the relationship.
- If the documents are distinct entities but merely related, keep their distinct facts and add useful [[WikiLinks]] between them.
- Do not delete useful entity-specific facts just because they are similar.
- Do not invent relationships or facts.
- Preserve useful history, technical specifics, structure, and existing [[WikiLinks]].
- Most current truth wins when facts conflict.
- Keep both documents concise and information-dense.
- No frontmatter, preamble, filler, or AI commentary.

Document A ("${node.name}"):
\`\`\`markdown
${stripHeader(node.content)}
\`\`\`

Document B ("${closest.name}"):
\`\`\`markdown
${stripHeader(closest.content)}
\`\`\``,
		});
		const a = store.find(node.name);
		const b = store.find(closest.name);
		if(!a || !b || !result?.aContent || !result?.bContent) return null;
		a.content = this.touchHeader(a, result.aContent);
		b.content = this.touchHeader(b, result.bContent);
		await Promise.all([embedMemoryFields(a, this.llm), embedMemoryFields(b, this.llm)]);
		return a;
	}

	private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
		const key = node.name;
		const existing = this.queues.get(key);
		if(existing) {
			existing.dirty = true;
			existing.request?.abort?.();
			return existing.task;
		}

		const entry = {dirty: false, request: null, task: Promise.resolve()};
		this.queues.set(key, entry);
		const store = new MemoryAccessor(memories);
		entry.task = (async () => {
			let current = node;
			try {
				do {
					entry.dirty = false;
					await this.docAgent(current, store.list, options, entry);
					this.mergeLock = this.mergeLock.then(() => this.mergeAgent(current, memories, options));
					const result = await this.mergeLock;
					if(result) current = result;
				} while(entry.dirty);
			} finally {
				store.commit([node]);
				this.queues.delete(key);
			}
		})();
		return entry.task;
	}

	private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
		if(!memories.includes(node)) return;
		const currentBody = stripHeader(node.content);
		const journal = node.name.startsWith('Journal/');
		const system = (journal
			? `You maintain one persistent journal document

Rewrite the ENTIRE journal, folding "## Pending" into the existing content removing the heading

Journal design:
- Preserve the chronological daily log
- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
- Group information by day under a date heading
- Keep journal entries high level and concise: what was worked on and the outcome, not a step-by-step record of how — that detail lives in conversation history, not here
- Use [[WikiLinks]] for persistent entities; don't turn ordinary journal events into entities
- No frontmatter, preamble, filler, or AI commentary`
			: `You maintain one persistent knowledge-base entity document

Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished.

Document design:
- The document represents one persistent entity. Keep information about that entity together and organized into sections
- Merge any pending information in, newest fact wins conflicts; remove redundant content
- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
- Let the structure fit the entity; there is NO fixed template
- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
- Keep the document concise and information-dense without removing useful technical specifics
- Current truth wins when facts conflict. Preserve older conflict as context, only when it adds useful meaning
- No frontmatter, preamble, filler, or AI commentary`) + `

Available nodes to link to:
${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}

Current document:
\`\`\`markdown
${currentBody}
\`\`\``;
		let update;
		try {
			for(let i = 0; i < 2 && !update?.content; i++) {
				const request = this.llm.ask(currentBody, {
					model: options.model,
					temperature: 0.3,
					schema: {
						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
					},
					system,
				});
				entry.request = request;
				update = await request;
			}
		} catch(err: any) {
			if(err?.name === 'AbortError') return;
			throw err;
		} finally {
			entry.request = null;
		}

		if(!update?.content) return;
		node.description = node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.name !== 'People/User' ? update.description.replaceAll(/[\n:]/g, '') : 'All information about the current user';
		node.content = this.touchHeader(node, update.content);
		await embedMemoryFields(node, this.llm);
	}

	private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
		const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
		if(!match) return {fm: new Map(), body: content};
		const fm = new Map<string, string>();
		for(const line of match[1].split('\n')) {
			const i = line.indexOf(':');
			if(i === -1) continue;
			const key = line.slice(0, i).trim();
			const raw = line.slice(i + 1).trim();
			let value = raw;
			try { value = JSON.parse(raw); } catch { }
			fm.set(key, value);
		}
		return {fm, body: match[2]};
	}

	private touchHeader(node: Memory, body: string): string {
		const {fm} = this.parseFrontmatter(node.content);
		fm.set('name', node.name);
		fm.set('description', (node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.description) || 'Persistent memory document');
		fm.set('modified', new Date().toISOString());
		return this.writeFrontmatter(fm, stripHeader(body));
	}

	private writeFrontmatter(fm: Map<string, string>, body: string): string {
		const lines = [...fm.entries()].map(([k, v]) => `${k}: ${JSON.stringify(String(v).replace(/\s+/g, ' ').trim())}`);
		return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`;
	}

	decay() {
		for(const [name, ttl] of this.recentlyTouched) {
			if(ttl <= 1) this.recentlyTouched.delete(name);
			else this.recentlyTouched.set(name, ttl - 1);
		}
	}

	touch(name: string, ttl = 2) {
		this.recentlyTouched.set(name, ttl);
	}

	forget(name: string, memories: Memory[] | MemoryCache): boolean {
		return new MemoryAccessor(memories).forget(name);
	}

	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
		function rank(query: number[], candidates: Memory[], limit: number): Memory[] {
			const scored = candidates.map(m => {
				const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0;
				const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0;
				const bodySim = m.bodyEmbeddings?.length
					? Math.max(...m.bodyEmbeddings.map(b => 1 - cosineDistance(query, b)))
					: 0;
				return {memory: m, score: titleSim * 0.5 + descSim * 0.35 + bodySim * 0.15};
			});
			return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory);
		}

		const store = new MemoryAccessor(memories);
		if(!store.list.length) return [];
		await store.backfillEmbeddings(this.llm);

		const [e] = await this.llm.embedding(query);
		if(!e) return [];

		const pool = store.search(e.embedding, Math.max(limit * 3, limit));
		const poolMemories = pool.map(r => store.find(r.name)).filter((m): m is Memory => !!m);
		const ranked = rank(e.embedding, poolMemories, limit);
		const found = new Set<string>(ranked.map(m => m.name));

		if(graphDepth > 0) {
			let frontier = [...found];
			for(let depth = 0; depth < graphDepth && frontier.length; depth++) {
				const next: string[] = [];
				for(const name of frontier) {
					const node = store.find(name);
					if(!node) continue;
					for(const link of node.links) {
						if(!found.has(link) && store.find(link)) {
							found.add(link);
							next.push(link);
						}
					}
				}
				frontier = next;
			}
		}

		const rankedOrder = ranked.map(m => m.name);
		const graphExpansions = [...found].filter(n => !rankedOrder.includes(n));
		return [...rankedOrder, ...graphExpansions].map(n => store.find(n)!).filter(Boolean);
	}

	async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory[]> {
		const conversation = history
			.filter(h => h.role === 'user' || h.role === 'assistant')
			.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
		if(!conversation) return [];

		const uid = `${Date.now()}_${Math.random().toString(36).slice(2)}`;
		const pending = {role: 'tool', name: 'memory_process', id: uid, content: conversation} as unknown as LLMMessage;
		history.push(pending);

		const store = new MemoryAccessor(memories);
		const {buckets, journal, tasks} = await this.factAgent(conversation, store, options);
		const touched: Memory[] = [];

		const personalTasks = tasks.filter(isPersonalTask);
		const entityTasks = tasks.filter(t => !isPersonalTask(t));

		if(journal || personalTasks.length) {
			const journalName = `Journal/${this.getWeekStart()}`;
			let jnode = store.find(journalName);
			const isNew = !jnode;
			if(!jnode) {
				jnode = {
					name: journalName,
					description: this.journalDescription(),
					content: '',
					embedding: [],
					links: [],
					backlinks: [],
				};
				store.list.push(jnode);
			}

			const blocks: string[] = [];
			if(journal) blocks.push(`### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
			if(isNew) {
				const previousDate = new Date(`${this.getWeekStart()}T00:00:00Z`);
				previousDate.setUTCDate(previousDate.getUTCDate() - 7);
				const previous = store.find(`Journal/${previousDate.toISOString().slice(0, 10)}`);
				if(previous) {
					const todos = this.getIncompleteTodos(previous.content);
					if(todos.length) blocks.push(`${TODO_HEADING}\n${todos.map(task => `- [ ] ${task}`).join('\n')}`);
				}
			}
			if(personalTasks.length) blocks.push(`${TODO_HEADING}\n${personalTasks.map(task => `- [${task.done ? 'x' : ' '}] ${task.task}`).join('\n')}`);
			if(blocks.length) this.stage(jnode, blocks.join('\n\n'));
			touched.push(jnode);
		}

		const entityStaging = new Map<string, {facts: string[], tasks: MemoryTask[]}>();
		for(const {subject, facts} of buckets) {
			const resolved = this.resolveSubject(subject, store);
			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
			entry.facts.push(...facts);
			entityStaging.set(resolved, entry);
		}
		for(const task of entityTasks) {
			const resolved = this.resolveSubject(task.subject, store);
			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
			entry.tasks.push(task);
			entityStaging.set(resolved, entry);
		}

		for(const [resolved, {facts, tasks: subjectTasks}] of entityStaging) {
			let node = store.find(resolved);
			if(!node) {
				node = {name: resolved, description: 'Persistent memory document', content: '', embedding: [], links: [], backlinks: []};
				store.list.push(node);
			}
			const blocks: string[] = [];
			if(facts.length) blocks.push(facts.map(f => `- ${f}`).join('\n'));
			if(subjectTasks.length) blocks.push(`${TODO_HEADING}\n${subjectTasks.map(t => `- [${t.done ? 'x' : ' '}] ${t.task}`).join('\n')}`);
			if(blocks.length) this.stage(node, blocks.join('\n\n'));
			touched.push(node);
		}

		await Promise.all(touched.map(async node => {
			await embedMemoryFields(node, this.llm);
			this.touch(node.name);
		}));

		if(touched.length) {
			store.commit(touched);
			(pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`;
			Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
		} else {
			(pending as any).content = 'Nothing worth remembering.';
		}

		(touched as any).uid = uid;
		return touched;
	}

	async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
		const store = new MemoryAccessor(memories);
		const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING));
		await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
		store.commit();
	}
}