Memorization prompt tightening

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.11",
+	"version": "1.7.1",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/llm.ts

@@ -19,6 +19,13 @@ const PDF_OCR_PAGE_THRESHOLD = 12; // above this many pages, OCR scanned pages i
 export type AnthropicConfig = {proto: 'anthropic', token: string | string[]};
 export type OpenAiConfig = {proto: 'openai', host?: string, token: string | string[]};
 
+export type AgentRef = {
+	name: string;
+	description?: string;
+	delegate?: boolean;
+	fn: () => Agent | null | Promise<Agent | null>;
+}
+
 export type Agent = {
 	name: string;
 	description?: string;
@@ -29,7 +36,7 @@ export type Agent = {
 	skills?: Skill[] | null;
 	tools?: AiTool[] | null;
 	mcp?: McpServer[] | null;
-	agents?: string[] | null;
+	agents?: AgentRef[] | null;
 }
 
 export type LLMFile = {
@@ -106,8 +113,8 @@ export type LLMRequest = {
 	skills?: Skill[];
 	/** MCP servers to connect and expose as tools */
 	mcp?: McpServer[];
-	/** Subagents exposed as delegatable/wrapped tools */
-	agents?: Agent[];
+	/** Subagents exposed as delegatable/wrapped tools, resolved lazily via their `fn` */
+	agents?: AgentRef[];
 	/** Attach files to request */
 	files?: LLMFile[];
 	/** @internal recursion guard for nested agent delegation */
@@ -265,22 +272,21 @@ class LLM {
 		};
 	}
 
-	private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: ((keep?: boolean) => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
-		return agents.map(a => {
-			const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`;
+	private setupAgent(stubs: AgentRef[] = [], history: LLMMessage[], aborts: ((keep?: boolean) => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
+		return stubs.map(stub => {
+			const toolName = `${stub.delegate ? '' : 'sub'}agent_${snakeCase(stub.name)}`;
 			return {
 				name: toolName,
-				description: `${a.delegate ? 'Delegate to ' : ''}Subagent: ${a.description || a.name}`,
+				description: `${stub.delegate ? 'Delegate to ' : ''}Subagent: ${stub.description || stub.name}`,
 				args: clean<any>({
-					context: !a.delegate ? {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true} : undefined,
+					context: !stub.delegate ? {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true} : undefined,
 					instructions: {type: 'string', description: 'Detailed instructions for subagent to complete', required: true},
 				}),
 				fn: async (args: any, stream: any, ai: any, id?: string) => {
 					if(depth >= MAX_AGENT_DEPTH) return 'Max agent delegation depth exceeded';
 
-					const nested = (a.agents || [])
-						.map(name => allAgents.find(x => x.name === name))
-						.filter((x): x is Agent => !!x && x.name !== a.name);
+					const a = await stub.fn();
+					if(!a) return `Agent "${stub.name}" could not be resolved`;
 
 					const q = a.delegate ? '' : `${args.instructions}${args.context ? `\n\n<context>${args.context}</context>` : ''}`;
 
@@ -297,7 +303,7 @@ ${a.system}`,
 						mcp: a.mcp || undefined,
 						skills: a.skills || undefined,
 						tools: a.tools || undefined,
-						agents: nested,
+						agents: a.agents || [],
 						_agentDepth: depth + 1,
 					} as any);
 					aborts.push(request.abort);
@@ -451,7 +457,7 @@ ${a.system}`,
 			// Agents
 			const agents = options.agents || this.ai.options?.llm?.agents;
 			const delegateState: {resp: string | null} = {resp: null};
-			if(agents?.length) tools.push(...this.setupAgent(agents, agents, history, nestedAborts, options._agentDepth || 0, delegateState));
+			if(agents?.length) tools.push(...this.setupAgent(agents, history, nestedAborts, options._agentDepth || 0, delegateState));
 
 			// Memory
 			const mem = MemoryManager.normalize(options.memory);

src/memory.ts

@@ -2,10 +2,10 @@ import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts';
 import {LLMRequest, LLMMessage} from './llm.ts';
 import {AiTool} from './tools.ts';
 import {KDTree} from './kd-tree.ts';
-import {escapeRegex} from '@ztimson/utils';
 
-const MERGE_THRESHOLD = 0.12;
+const FACT_SIMILARITY_THRESHOLD = 0.62;
 const PENDING_HEADING = '## Pending';
+const TODO_HEADING = '## Todo list';
 const TREE_TOMBSTONE_LIMIT = 0.25;
 const ALIAS_MATCH_THRESHOLD = 0.55;
 
@@ -13,11 +13,8 @@ export type Memory = {
 	name: string;
 	description: string;
 	content: string;
-	/** Description embedding — indexed in the KD tree, used for merge/ANN candidate lookup */
 	embedding: number[];
-	/** Title-only embedding, weighted heaviest during recall ranking */
 	titleEmbedding?: number[];
-	/** Chunked body embeddings, best-chunk match used during recall ranking */
 	bodyEmbeddings?: number[][];
 	links: string[];
 	backlinks: string[];
@@ -26,7 +23,6 @@ export type Memory = {
 type MemoryRef = {
 	name: string;
 	description: string;
-	/** Cosine distance from the query, present when returned from a search */
 	distance?: number;
 }
 
@@ -35,9 +31,17 @@ type FactBucket = {
 	facts: string[];
 }
 
+type MemoryTask = {
+	/** Exact node name / new persistent entity path this task belongs to, or '' for a personal task with no entity (goes to the journal) */
+	subject: string;
+	task: string;
+	done: boolean;
+}
+
 type FactAgentResult = {
 	buckets: FactBucket[];
 	journal: string;
+	tasks: MemoryTask[];
 }
 
 function dedupeFacts(facts: string[]): string[] {
@@ -68,7 +72,6 @@ function cosineSearch(query: number[], memories: Memory[], limit: number): Memor
 		.slice(0, limit);
 }
 
-/** Re-embed a node's title / description / body fields. Description embedding stays the KD-tree index key. */
 async function embedMemoryFields(node: Memory, llm: any): Promise<void> {
 	const body = stripHeader(node.content);
 	const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name);
@@ -83,9 +86,14 @@ export function stripHeader(content: string): string {
 	return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
 }
 
+/** True if a task has no persistent entity of its own and belongs in the journal instead. */
+function isPersonalTask(t: MemoryTask): boolean {
+	const s = (t.subject ?? '').trim().toLowerCase();
+	return !s || s === 'journal' || s.startsWith('journal/');
+}
+
 export class MemoryCache {
 	private tree!: KDTree<MemoryRef>;
-	/** Tracks which memories are currently indexed in the tree, keyed by name -> embedding reference */
 	private indexed = new Map<string, number[]>();
 	public memories: Memory[];
 	public nodes: MemoryNode[] = [];
@@ -98,7 +106,6 @@ export class MemoryCache {
 		this.rebuild();
 	}
 
-	/** Incrementally sync the KD tree against `this.memories` instead of rebuilding from scratch */
 	private syncTree(): void {
 		const current = new Set(this.memories.map(m => m.name));
 
@@ -180,7 +187,6 @@ class MemoryAccessor {
 		return nodes.filter(n => n.missing).map(n => n.name);
 	}
 
-	/** Cache path uses the KD tree's knn(); raw-array path (no cache available) falls back to a linear cosine scan */
 	search(vector: number[], limit: number): MemoryRef[] {
 		return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit);
 	}
@@ -241,10 +247,10 @@ export class MemoryManager {
 			name: 'memory_recall',
 			description: 'Read the full content of a memory document',
 			args: {
-				name: {type: 'string', description: 'Exact memory name', required: true},
+				name: {type: 'string', description: 'Exact memory name', required: true}
 			},
 			fn: (args: any) => {
-				const mem = this.access(memories).find(args.name);
+				const mem = new MemoryAccessor(memories).find(args.name);
 				if(!mem) return 'Document not found';
 				this.touch(mem.name);
 				return mem.content;
@@ -259,7 +265,7 @@ export class MemoryManager {
 				limit: {type: 'number', description: 'Number of memories to return', default: 1},
 			},
 			fn: async ({query, limit}) => {
-				const mem = await this.recollect(query, memories, limit)
+				const mem = await this.recollect(query, memories, limit);
 				return mem.map(m => `Memory: ${m.name}
 Description: ${m.description}
 Links: ${[...m.links, ...m.backlinks].join(', ')}
@@ -278,12 +284,11 @@ ${m.content}
 		return raw ? {memory: <Memory[] | MemoryCache>m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m};
 	}
 
-	private access(memories: Memory[] | MemoryCache): MemoryAccessor {
-		return new MemoryAccessor(memories);
-	}
-
 	private stage(node: Memory, block: string): void {
-		this.ensureDoc(node);
+		if(!node.content) {
+			const title = node.name.split('/').pop() ?? node.name;
+			node.content = this.touchHeader(node, `# ${title}\n`);
+		}
 		const body = stripHeader(node.content);
 		const idx = body.indexOf(PENDING_HEADING);
 		const newBody = idx === -1
@@ -292,38 +297,17 @@ ${m.content}
 		node.content = this.touchHeader(node, newBody);
 	}
 
-	private ensureDoc(node: Memory): void {
-		if (node.content) return;
-		const title = node.name.split('/').pop() ?? node.name;
-		node.content = this.touchHeader(node, `# ${title}\n`);
-	}
-
-	private sanitizeDescription(text: string): string {
-		return (text ?? '').replace(/\s+/g, ' ').trim().slice(0, 240);
-	}
-
-	private relink(memories: Memory[], from: string, to: string): void {
-		const pattern = new RegExp(`\\[\\[${escapeRegex(from)}\\]\\]`, 'g');
-		for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`);
-	}
-
-	private normalizeLeaf(name: string): string {
+	private resolveSubject(subject: string, store: MemoryAccessor): string {
+		function normalize(name: string): string {
 			return name.trim().toLowerCase().replace(/\s+/g, ' ');
 		}
 
-	/**
-	 * Resolve a fact-agent proposed subject to an existing node when it's an alias/rename of one.
-	 * Exact match is checked first (cheap, and covers the common case since node names are
-	 * already normalized at creation time). Only falls through to fuzzy alias matching against
-	 * same-root candidates when there's no existing hit — i.e. only on likely-new-doc creation.
-	 */
-	private resolveSubject(subject: string, store: MemoryAccessor): string {
 		const trimmed = subject.trim();
 		const exact = store.find(trimmed);
 		if(exact) return exact.name;
 
-		const normalized = this.normalizeLeaf(trimmed);
-		const caseInsensitive = store.list.find(m => this.normalizeLeaf(m.name) === normalized);
+		const normalized = normalize(trimmed);
+		const caseInsensitive = store.list.find(m => normalize(m.name) === normalized);
 		if(caseInsensitive) return caseInsensitive.name;
 
 		const root = trimmed.split('/')[0];
@@ -345,32 +329,48 @@ ${m.content}
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
-			system: `Extract durable memory from this conversation
+			system: `Turn this conversation into a persistent memory file by extracting information into organized bullet points
 
-1. Journal recap
-- Brief "Captain's Log" of what happened, including useful context, decisions, or events
-- Leave empty for trivial exchanges
+Think of this like an Obsidian vault with a clear division of responsibility:
+- The JOURNAL is a timeline. It answers "what happened, and when" and is the only place with a sense of time.
+- ENTITY DOSSIERS are a wiki. They answer "what is currently true about this subject", with no sense of time — only current state.
+- Never blur the two: a one-off event, conversation, or debugging session is a journal entry, not an entity, even if it's detailed.
 
-2. Fact buckets
-- Extract only durable facts explicitly stated by the USER
+1. Journal Log
+- A chronological, skimmable log of what actually happened: real discussions, decisions made, progress on projects, problems worked through
+- This is NOT a transcript, and it is NOT a step-by-step record, its a compressed log of notable events & developments
+- One line per development is usually enough: what was worked on and the outcome, not the blow-by-blow of how
+- Skip small talk and trivial exchanges entirely. Skip anything that's a todo item (goes in Todo Tasks) or a durable fact about a subject (goes in Entity Dossiers)
+
+2. Todo Tasks
+- Extract concrete tasks the user says need to be done, should be done, or were completed
+- Return the task text and whether it is still todo or is done
+- A completed task should be marked done, not recreated as a new todo
+- Only extract actionable tasks, not general goals or observations, if none - omit returning a tasks array
+- Assign each task a subject:
+    - If the task belongs to a persistent entity (a project, a class, etc.), use that entity's exact node name, or a new entity path if it doesn't exist yet
+    - If it's a personal/life task with no entity of its own (reach out to someone, reply to an email, pay a bill, etc.), leave subject as an empty string — it belongs in the journal, not a new document
+
+3. Entity Dossiers
+- Detailed dossiers with all factual information regarding a subject
 - Record the final/end state, not intermediate changes
-- Do not extract assistant claims, guesses, greetings, or temporary conversation details
-
-For each fact, identify its HOME ENTITY:
+- Ignore assistant claims, guesses, greetings, or temporary details
+- NEVER create a dossier for something I wouldnt find in a wiki site: temporary information, debugging, guesses, conversations (this is all journal entry stuff!)
+- identify its HOME ENTITY:
     - The HOME ENTITY name should always be a [abstract|pro]noun
     - The grammatical subject/owner of the fact is the strongest clue
-- Prefer an existing entity over creating a new one
+    - Always preference an existing entity over creating a new one
+    - New child entities are appropriate only when they are themselves distinct persistent entities
     - A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment
     - Put project facts under the project they belong to, person facts under the person, etc
-- New child entities are appropriate only when they are themselves distinct persistent entities
 
-Example Paths:
+Example Entity Naming Convention:
 - Projects/[Name]
 - People/[Name]
 - History/[Name]
 - Science/[Name]
 - [Subject]/[Name]
-- Class/[Name]/[Child]
+- Class/[Name]/[Chapter]
 
 Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships
 Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent
@@ -379,8 +379,18 @@ Available nodes:
 ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
 ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			schema: {
-				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
-				buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
+				journal: {type: 'string', description: 'Short bullet point recap, omit if nothing notable happened'},
+				tasks: {
+					type: 'array', description: 'Concrete tasks mentioned or completed in the conversation, omit if none', items: {
+						type: 'object', items: {
+							subject: {type: 'string', description: 'Exact node name / new persistent entity path this task belongs to, or an empty string if this is a personal task with no entity of its own (those go in the journal)', required: true},
+							task: {type: 'string', description: 'Concise actionable task', required: true},
+							done: {type: 'boolean', description: 'Whether the task is completed', required: true},
+						},
+					}
+				},
+				buckets: {
+					type: 'array', description: 'Groups of facts to remember; omit if none', items: {
 						type: 'object', items: {
 							subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true},
 							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
@@ -401,10 +411,11 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return {
 			buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
 			journal: (response.journal ?? '').trim(),
+			tasks: response.tasks ?? [],
 		};
 	}
 
-	private getWeekMonday(date: Date = new Date()): string {
+	private getWeekStart(date: Date = new Date()): string {
 		const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
 		const day = d.getUTCDay();
 		const diff = day === 0 ? -6 : 1 - day;
@@ -412,38 +423,86 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return d.toISOString().slice(0, 10);
 	}
 
+	private journalDescription(journalName?: string): string {
+		const start = journalName?.split('/').pop() || this.getWeekStart();
+		const d = new Date(`${start}T00:00:00Z`);
+		d.setUTCDate(d.getUTCDate() + 6);
+		const end = d.toISOString().slice(0, 10);
+		return `Log from ${start} - ${end}`;
+	}
+
+	private getIncompleteTodos(content: string): string[] {
+		const body = stripHeader(content);
+		const match = body.match(/## Todo list\n([\s\S]*?)(?=\n## |$)/i);
+		if(!match) return [];
+		return match[1].split('\n')
+			.map(line => line.match(/^\s*-\s*\[([ xX])\]\s+(.+?)\s*$/))
+			.filter((m): m is RegExpMatchArray => !!m && m[1].toLowerCase() !== 'x')
+			.map(m => m[2].trim());
+	}
+
 	private listNodes(memories: Memory[]): MemoryRef[] {
 		return memories.map(m => ({name: m.name, description: m.description}));
 	}
 
-	/** Finds the closest merge candidate via the KD tree's knn() instead of a manual O(n) cosine scan */
-	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
+	private async mergeAgent(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory | null> {
+		function factSimilarity(a: Memory, b: Memory): number {
+			if(!a.bodyEmbeddings?.length || !b.bodyEmbeddings?.length) return 0;
+			let best = 0;
+			for(const av of a.bodyEmbeddings) {
+				for(const bv of b.bodyEmbeddings) best = Math.max(best, 1 - cosineDistance(av, bv));
+			}
+			return best;
+		}
+
 		if(!node.embedding?.length || node.name.startsWith('Journal/')) return null;
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
+		const candidates = store.list
+			.filter(m => m.name !== node.name && !m.name.startsWith('Journal/'))
+			.filter(m => factSimilarity(node, m) >= FACT_SIMILARITY_THRESHOLD);
 
-		const candidate = store.search(node.embedding, 5)
-			.find(r => r.name !== node.name && !r.name.startsWith('Journal/') && r.distance !== undefined && r.distance <= threshold);
-		if (!candidate) return null;
-		const closest = store.find(candidate.name);
-		if (!closest) return null;
+		if(!candidates.length) return null;
+		const closest = candidates.sort((a, b) => factSimilarity(node, b) - factSimilarity(node, a))[0];
+		const result = await this.llm.ask('', {
+			model: options.model,
+			temperature: 0.3,
+			schema: {
+				aContent: {type: 'string', description: 'Updated document A body in markdown, without frontmatter.', required: true},
+				bContent: {type: 'string', description: 'Updated document B body in markdown, without frontmatter.', required: true},
+			},
+			system: `Maintain these two persistent knowledge-base documents like a wiki.
 
-		const result = await this.mergeAgent(node, closest, options);
-		const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []};
-		merged.content = this.touchHeader(merged, result.content);
-		await embedMemoryFields(merged, this.llm);
+Do NOT merge, rename, or delete either document. Both represent entities that should remain independently addressable.
 
-		this.relink(store.list, node.name, merged.name);
-		this.relink(store.list, closest.name, merged.name);
+The documents were selected because their facts may overlap. Your job is to reconcile duplicated information and connect the documents:
+- Decide which document is the HOME for each duplicated fact.
+- Keep the authoritative copy in that home document.
+- In the other document, replace the information with a short preamble and [[WikiLink]] to the home entity explaining the relationship.
+- If the documents are distinct entities but merely related, keep their distinct facts and add useful [[WikiLinks]] between them.
+- Do not delete useful entity-specific facts just because they are similar.
+- Do not invent relationships or facts.
+- Preserve useful history, technical specifics, structure, and existing [[WikiLinks]].
+- Most current truth wins when facts conflict.
+- Keep both documents concise and information-dense.
+- No frontmatter, preamble, filler, or AI commentary.
 
-		this.queues.get(closest.name)?.request?.abort?.();
-		this.queues.delete(closest.name);
+Document A ("${node.name}"):
+\`\`\`markdown
+${stripHeader(node.content)}
+\`\`\`
 
-		store.forget(node.name);
-		store.forget(closest.name);
-		store.list.push(merged);
-		store.commit();
-
-		return merged;
+Document B ("${closest.name}"):
+\`\`\`markdown
+${stripHeader(closest.content)}
+\`\`\``,
+		});
+		const a = store.find(node.name);
+		const b = store.find(closest.name);
+		if(!a || !b || !result?.aContent || !result?.bContent) return null;
+		a.content = this.touchHeader(a, result.aContent);
+		b.content = this.touchHeader(b, result.bContent);
+		await Promise.all([embedMemoryFields(a, this.llm), embedMemoryFields(b, this.llm)]);
+		return a;
 	}
 
 	private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
@@ -457,19 +516,19 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 
 		const entry = {dirty: false, request: null, task: Promise.resolve()};
 		this.queues.set(key, entry);
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		entry.task = (async () => {
-			let current = node, merged = false;
+			let current = node;
 			try {
 				do {
 					entry.dirty = false;
 					await this.docAgent(current, store.list, options, entry);
-					this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
+					this.mergeLock = this.mergeLock.then(() => this.mergeAgent(current, memories, options));
 					const result = await this.mergeLock;
-					if (result) { current = result; merged = true; }
+					if(result) current = result;
 				} while(entry.dirty);
 			} finally {
-				store.commit(merged ? undefined : [node]);
+				store.commit([node]);
 				this.queues.delete(key);
 			}
 		})();
@@ -479,6 +538,40 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 	private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
 		if(!memories.includes(node)) return;
 		const currentBody = stripHeader(node.content);
+		const journal = node.name.startsWith('Journal/');
+		const system = (journal
+			? `You maintain one persistent journal document
+
+Rewrite the ENTIRE journal, folding "## Pending" into the existing content removing the heading
+
+Journal design:
+- Preserve the chronological daily log
+- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
+- Group information by day under a date heading
+- Keep journal entries high level and concise: what was worked on and the outcome, not a step-by-step record of how — that detail lives in conversation history, not here
+- Use [[WikiLinks]] for persistent entities; don't turn ordinary journal events into entities
+- No frontmatter, preamble, filler, or AI commentary`
+			: `You maintain one persistent knowledge-base entity document
+
+Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished.
+
+Document design:
+- The document represents one persistent entity. Keep information about that entity together and organized into sections
+- Merge any pending information in, newest fact wins conflicts; remove redundant content
+- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
+- Let the structure fit the entity; there is NO fixed template
+- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
+- Keep the document concise and information-dense without removing useful technical specifics
+- Current truth wins when facts conflict. Preserve older conflict as context, only when it adds useful meaning
+- No frontmatter, preamble, filler, or AI commentary`) + `
+
+Available nodes to link to:
+${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
+
+Current document:
+\`\`\`markdown
+${currentBody}
+\`\`\``;
 		let update;
 		try {
 			for(let i = 0; i < 2 && !update?.content; i++) {
@@ -489,28 +582,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
-					system: `You maintain one persistent knowledge-base document
-
-Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished
-
-Document design:
-- The document represents one entity. Keep information about that entity together
-- Let the structure fit the entity; there is NO fixed template
-- Preserve useful existing headings and organization. Don't redesign the document without reason
-- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
-- Keep the document concise and information-dense without removing useful technical specifics
-- Current truth wins when facts conflict. Preserve older context only when it adds useful meaning
-- Use [[WikiLinks]] for specific related entities; don't create redundant content for linked entities
-- Avoid generic filler sections such as Notes, Miscellaneous, Recent, Updates, or Conversation
-- No frontmatter, preamble, filler, or AI commentary
-
-Available nodes to link to:
-${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
-
-Current document:
-\`\`\`markdown
-${currentBody}
-\`\`\``,
+					system,
 				});
 				entry.request = request;
 				update = await request;
@@ -523,47 +595,11 @@ ${currentBody}
 		}
 
 		if(!update?.content) return;
-		node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
+		node.description = node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.name !== 'People/User' ? update.description.replaceAll(/[\n:]/g, '') : 'All information about the current user';
 		node.content = this.touchHeader(node, update.content);
 		await embedMemoryFields(node, this.llm);
 	}
 
-	private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> {
-		const modifiedOf = (m: Memory) => this.parseFrontmatter(m.content).fm.get('modified') || 'unknown';
-
-		return this.llm.ask('', {
-			model: options.model,
-			temperature: 0.3,
-			schema: {
-				name: {type: 'string', description: 'Canonical path for the merged entity', required: true},
-				description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
-				content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
-			},
-			system: `Determine whether these two documents represent the SAME persistent entity.
-
-Similarity of subject matter is NOT enough. Do not merge documents merely because they discuss the same project, person, technology, topic, or related work.
-
-Merge only when the evidence indicates they are duplicate identities, aliases, renamed entities, or two documents accidentally created for the same real-world entity. If they are distinct entities, they must remain separate.
-
-If they are the same entity:
-- Choose the canonical/most established path.
-- Combine their information into one document and remove duplication.
-- Preserve useful structure, technical specifics, history, and [[WikiLinks]].
-- Prefer newer information when facts conflict.
-- Return the canonical entity name and the fully reconciled document.
-
-Document A ("${a.name}", last modified ${modifiedOf(a)}):
-\`\`\`markdown
-${stripHeader(a.content)}
-\`\`\`
-
-Document B ("${b.name}", last modified ${modifiedOf(b)}):
-\`\`\`markdown
-${stripHeader(b.content)}
-\`\`\``,
-		});
-	}
-
 	private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
 		const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
 		if(!match) return {fm: new Map(), body: content};
@@ -574,21 +610,16 @@ ${stripHeader(b.content)}
 			const key = line.slice(0, i).trim();
 			const raw = line.slice(i + 1).trim();
 			let value = raw;
-			try { value = JSON.parse(raw); } catch { /* legacy unquoted value, keep raw */ }
+			try { value = JSON.parse(raw); } catch { }
 			fm.set(key, value);
 		}
 		return {fm, body: match[2]};
 	}
 
-	/**
-	 * Writes the code-owned frontmatter block. `body` is passed through stripHeader() first so a
-	 * model that ignores instructions and hallucinates its own `---` block can never corrupt or
-	 * duplicate the real frontmatter — the LLM only ever gets to influence the body.
-	 */
 	private touchHeader(node: Memory, body: string): string {
 		const {fm} = this.parseFrontmatter(node.content);
 		fm.set('name', node.name);
-		fm.set('description', node.description || '');
+		fm.set('description', (node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.description) || 'Persistent memory document');
 		fm.set('modified', new Date().toISOString());
 		return this.writeFrontmatter(fm, stripHeader(body));
 	}
@@ -610,11 +641,11 @@ ${stripHeader(b.content)}
 	}
 
 	forget(name: string, memories: Memory[] | MemoryCache): boolean {
-		return this.access(memories).forget(name);
+		return new MemoryAccessor(memories).forget(name);
 	}
 
-	/** Ranks a candidate pool by weighted title/description/body similarity against the query embedding */
-	private rankByFields(query: number[], candidates: Memory[], limit: number): Memory[] {
+	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
+		function rank(query: number[], candidates: Memory[], limit: number): Memory[] {
 			const scored = candidates.map(m => {
 				const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0;
 				const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0;
@@ -626,19 +657,16 @@ ${stripHeader(b.content)}
 			return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory);
 		}
 
-	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		if(!store.list.length) return [];
-
 		await store.backfillEmbeddings(this.llm);
 
 		const [e] = await this.llm.embedding(query);
 		if(!e) return [];
 
-		// Description embedding is the cheap ANN index key; pull a wider pool then re-rank by field weight
 		const pool = store.search(e.embedding, Math.max(limit * 3, limit));
 		const poolMemories = pool.map(r => store.find(r.name)).filter((m): m is Memory => !!m);
-		const ranked = this.rankByFields(e.embedding, poolMemories, limit);
+		const ranked = rank(e.embedding, poolMemories, limit);
 		const found = new Set<string>(ranked.map(m => m.name));
 
 		if(graphDepth > 0) {
@@ -674,29 +702,69 @@ ${stripHeader(b.content)}
 		const pending = {role: 'tool', name: 'memory_process', id: uid, content: conversation} as unknown as LLMMessage;
 		history.push(pending);
 
-		const store = this.access(memories);
-		const {buckets, journal} = await this.factAgent(conversation, store, options);
+		const store = new MemoryAccessor(memories);
+		const {buckets, journal, tasks} = await this.factAgent(conversation, store, options);
 		const touched: Memory[] = [];
 
-		if (journal) {
-			const journalName = `Journal/${this.getWeekMonday()}`;
+		const personalTasks = tasks.filter(isPersonalTask);
+		const entityTasks = tasks.filter(t => !isPersonalTask(t));
+
+		if(journal || personalTasks.length) {
+			const journalName = `Journal/${this.getWeekStart()}`;
 			let jnode = store.find(journalName);
+			const isNew = !jnode;
 			if(!jnode) {
-				jnode = {name: journalName, description: '', content: '', embedding: [], links: [], backlinks: []};
+				jnode = {
+					name: journalName,
+					description: this.journalDescription(),
+					content: '',
+					embedding: [],
+					links: [],
+					backlinks: [],
+				};
 				store.list.push(jnode);
 			}
-			this.stage(jnode, `### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+
+			const blocks: string[] = [];
+			if(journal) blocks.push(`### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+			if(isNew) {
+				const previousDate = new Date(`${this.getWeekStart()}T00:00:00Z`);
+				previousDate.setUTCDate(previousDate.getUTCDate() - 7);
+				const previous = store.find(`Journal/${previousDate.toISOString().slice(0, 10)}`);
+				if(previous) {
+					const todos = this.getIncompleteTodos(previous.content);
+					if(todos.length) blocks.push(`${TODO_HEADING}\n${todos.map(task => `- [ ] ${task}`).join('\n')}`);
+				}
+			}
+			if(personalTasks.length) blocks.push(`${TODO_HEADING}\n${personalTasks.map(task => `- [${task.done ? 'x' : ' '}] ${task.task}`).join('\n')}`);
+			if(blocks.length) this.stage(jnode, blocks.join('\n\n'));
 			touched.push(jnode);
 		}
 
+		const entityStaging = new Map<string, {facts: string[], tasks: MemoryTask[]}>();
 		for(const {subject, facts} of buckets) {
 			const resolved = this.resolveSubject(subject, store);
+			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
+			entry.facts.push(...facts);
+			entityStaging.set(resolved, entry);
+		}
+		for(const task of entityTasks) {
+			const resolved = this.resolveSubject(task.subject, store);
+			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
+			entry.tasks.push(task);
+			entityStaging.set(resolved, entry);
+		}
+
+		for(const [resolved, {facts, tasks: subjectTasks}] of entityStaging) {
 			let node = store.find(resolved);
 			if(!node) {
-				node = {name: resolved, description: '', content: '', embedding: [], links: [], backlinks: []};
+				node = {name: resolved, description: 'Persistent memory document', content: '', embedding: [], links: [], backlinks: []};
 				store.list.push(node);
 			}
-			this.stage(node, facts.map(f => `- ${f}`).join('\n'));
+			const blocks: string[] = [];
+			if(facts.length) blocks.push(facts.map(f => `- ${f}`).join('\n'));
+			if(subjectTasks.length) blocks.push(`${TODO_HEADING}\n${subjectTasks.map(t => `- [${t.done ? 'x' : ' '}] ${t.task}`).join('\n')}`);
+			if(blocks.length) this.stage(node, blocks.join('\n\n'));
 			touched.push(node);
 		}
 
@@ -718,7 +786,7 @@ ${stripHeader(b.content)}
 	}
 
 	async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING));
 		await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
 		store.commit();

src/open-ai.ts

@@ -36,21 +36,49 @@ export class OpenAi extends LLMProvider {
 	private toWire(history: LLMMessage[], system?: string): any[] {
 		const wire: any[] = [];
 		if(system) wire.push({role: 'system', content: system});
-		for(const h of history) {
-			if(h.role === 'tool') {
+
+		for(let i = 0; i < history.length; i++) {
+			const h = history[i];
+
+			if(h.role !== 'tool') {
+				wire.push({role: h.role, content: this.toWireContent(h.content)});
+				continue;
+			}
+
+			const calls: any[] = [];
+			const results: any[] = [];
+
+			while(i < history.length && history[i].role === 'tool') {
+				const tool: any = history[i];
+
+				calls.push({
+					id: tool.id,
+					type: 'function',
+					function: {
+						name: tool.name,
+						arguments: JSON.stringify(tool.args || {})
+					}
+				});
+
+				results.push({
+					role: 'tool',
+					tool_call_id: tool.id,
+					content: tool.error || tool.content || ''
+				});
+
+				i++;
+			}
+
 			wire.push({
 				role: 'assistant',
 				content: null,
-					tool_calls: [{id: h.id, type: 'function', function: {name: h.name, arguments: JSON.stringify(h.args)}}],
-				}, {
-					role: 'tool',
-					tool_call_id: h.id,
-					content: h.error || h.content || '',
+				tool_calls: calls
 			});
-			} else {
-				wire.push({role: h.role, content: this.toWireContent(h.content)});
-			}
+
+			wire.push(...results);
+			i--;
 		}
+
 		return wire;
 	}
 
@@ -60,13 +88,12 @@ export class OpenAi extends LLMProvider {
 			if(!options.history) options.history = [];
 			const history = options.history;
 			if(message) history.push({role: 'user', content: message, timestamp: Date.now()});
-
 			const tools = options.tools || this.ai.options.llm?.tools || [];
 			const requestParams: any = {
 				model: options.model || this.model,
 				stream: !!options.stream,
-				max_completion_tokens: options.maxTokens || this.ai.options.llm?.maxTokens || undefined,
-				temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
+				max_completion_tokens: options.maxTokens ?? this.ai.options.llm?.maxTokens,
+				temperature: options.temperature ?? this.ai.options.llm?.temperature,
 				tools: tools.map(t => ({
 					type: 'function',
 					function: {
@@ -74,8 +101,12 @@ export class OpenAi extends LLMProvider {
 						description: t.description,
 						parameters: {
 							type: 'object',
-							properties: t.args ? objectMap(t.args, (key, value) => ({...value, required: undefined})) : {},
-							required: t.args ? Object.entries(t.args).filter(t => t[1].required).map(t => t[0]) : []
+							properties: t.args
+								? objectMap(t.args, (key, value) => ({...value, required: undefined}))
+								: {},
+							required: t.args
+								? Object.entries(t.args).filter(t => t[1].required).map(t => t[0])
+								: []
 						}
 					}
 				}))
@@ -83,55 +114,79 @@ export class OpenAi extends LLMProvider {
 
 			if(options.schema) {
 				const schema = convertSchema(options.schema);
-				requestParams.response_format = {type: 'json_schema', json_schema: {name: 'response', strict: true, schema}};
+				requestParams.response_format = {
+					type: 'json_schema',
+					json_schema: {name: 'response', strict: true, schema}
+				};
 			}
 			if(options.stream) requestParams.stream_options = {include_usage: true};
 
 			try {
 				let terminal = false;
+				let iteration = 0;
+
 				do {
+					iteration++;
 					requestParams.messages = this.toWire(history.filter(h => h.role !== 'system'), options.system);
 
 					const callStart = Date.now();
-					const resp: any = await this.tokenPool.run(token => this.getClient(token).chat.completions.create(requestParams)).catch(err => {
+					const resp: any = await this.tokenPool.run(token =>
+						this.getClient(token).chat.completions.create(requestParams)
+					).catch(err => {
 						err.message += `\n\nMessages:\n${JSON.stringify(requestParams.messages, null, 2)}`;
 						throw err;
 					});
 
-					let usage: any, finishReason: string | undefined, msg: any = {content: '', tool_calls: []};
+					let usage: any;
+					let finishReason: string | undefined;
+					let msg: any = {content: '', tool_calls: []};
+					let streamedChars = 0;
+
 					if(options.stream) {
+						let streamCompleted = false;
+						try {
 							for await (const chunk of resp) {
 								if(controller.signal.aborted) break;
 								if(chunk.usage) usage = chunk.usage;
-							if(chunk.choices[0]?.finish_reason) finishReason = chunk.choices[0].finish_reason;
-							if(chunk.choices[0]?.delta?.content) {
-								msg.content += chunk.choices[0].delta.content;
-								options.stream({text: chunk.choices[0].delta.content});
+
+								const choice = chunk.choices?.[0];
+								if(choice?.finish_reason) finishReason = choice.finish_reason;
+
+								if(choice?.delta?.content) {
+									msg.content += choice.delta.content;
+									streamedChars += choice.delta.content.length;
+									options.stream({text: choice.delta.content});
 								}
-							if(chunk.choices[0]?.delta?.tool_calls) {
-								for(const deltaTC of chunk.choices[0].delta.tool_calls) {
-									const existing = deltaTC.index != null
-										? msg.tool_calls.find((tc: any) => tc.index === deltaTC.index)
-										: (deltaTC.id ? msg.tool_calls.find((tc: any) => tc.id === deltaTC.id) : undefined);
-									if(existing) {
+
+								if(choice?.delta?.tool_calls) {
+									for(const deltaTC of choice.delta.tool_calls) {
+										const index = deltaTC.index ?? msg.tool_calls.length;
+										let existing = msg.tool_calls.find((tc: any) => tc.index === index);
+
+										if(!existing) {
+											existing = {index, id: '', function: {name: '', arguments: ''}};
+											msg.tool_calls.push(existing);
+										}
+
 										if(deltaTC.id) existing.id = deltaTC.id;
 										if(deltaTC.function?.name) existing.function.name = deltaTC.function.name;
 										if(deltaTC.function?.arguments) existing.function.arguments += deltaTC.function.arguments;
-									} else {
-										msg.tool_calls.push({
-											index: deltaTC.index,
-											id: deltaTC.id || '',
-											function: {name: deltaTC.function?.name || '', arguments: deltaTC.function?.arguments || ''}
-										});
 									}
 								}
 							}
+
+							streamCompleted = true;
+						} catch(err) {
+							if(!controller.signal.aborted) throw err;
 						}
+
+						if(streamCompleted && !finishReason) finishReason = msg.tool_calls.length ? 'tool_calls' : 'stop';
 					} else {
 						usage = resp.usage;
 						finishReason = resp.choices[0].finish_reason;
 						msg = resp.choices[0].message;
 					}
+
 					const duration = Date.now() - callStart;
 					const tps = usage?.completion_tokens && duration > 0 ? usage.completion_tokens / (duration / 1000) : 0;
 
@@ -141,15 +196,24 @@ export class OpenAi extends LLMProvider {
 					}
 
 					if(!finishReason && !controller.signal.aborted) {
-						throw new Error('[OpenAI] Stream ended prematurely - connection likely dropped');
+						throw new Error('[OpenAI] Completion ended without a usable response');
 					}
 
 					const toolCalls = msg.tool_calls || [];
+
 					if(toolCalls.length && !controller.signal.aborted) {
 						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
 
 						const entries = toolCalls.map((tc: any) => {
-							const entry: any = {role: 'tool', id: tc.id, name: tc.function.name, args: JSONAttemptParse(tc.function.arguments, {}), content: undefined, timestamp: Date.now()};
+							const entry: any = {
+								role: 'tool',
+								id: tc.id,
+								name: tc.function.name,
+								args: JSONAttemptParse(tc.function.arguments, {}),
+								content: undefined,
+								timestamp: Date.now()
+							};
+
 							history.push(entry);
 							return {tc, entry};
 						});
@@ -157,12 +221,13 @@ export class OpenAi extends LLMProvider {
 						await Promise.all(entries.map(async ({tc, entry}: any) => {
 							const tool = tools.find(findByProp('name', tc.function.name));
 							if(options.stream) options.stream({tool: tc.function.name});
-							if(!tool) { entry.error = 'Tool not found'; return; }
+							if(!tool) return entry.error = 'Tool not found';
 							try {
 								const toolStream = options.stream && ((chunk: any) => {
 									if(chunk.done) return;
 									options.stream!(chunk);
 								});
+
 								const result = await tool.fn(entry.args, toolStream, this.ai, tc.id);
 								entry.content = typeof result === 'object' ? JSONSanitize(result) : result;
 							} catch(err: any) {
@@ -177,7 +242,6 @@ export class OpenAi extends LLMProvider {
 				} while(!terminal && !controller.signal.aborted);
 
 				if(options.stream) options.stream({done: true});
-
 				const turnStart = history.map(h => h.role).lastIndexOf('user');
 				const finalContent = history.slice(turnStart + 1).reduce((str, h) => h.role === 'assistant' ? str + (h.content || '') : str, '').trim();
 				res(options.schema ? JSONAttemptParse(finalContent, finalContent) : finalContent);