Fix open-ai early termination & memory improvements

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.7",
+	"version": "1.6.12",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/memory.ts

@@ -2,29 +2,19 @@ import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts';
 import {LLMRequest, LLMMessage} from './llm.ts';
 import {AiTool} from './tools.ts';
 import {KDTree} from './kd-tree.ts';
-import {escapeRegex} from '@ztimson/utils';
 
-const MERGE_THRESHOLD = 0.12;
+const FACT_SIMILARITY_THRESHOLD = 0.62;
 const PENDING_HEADING = '## Pending';
+const TODO_HEADING = '## Todo list';
 const TREE_TOMBSTONE_LIMIT = 0.25;
 const ALIAS_MATCH_THRESHOLD = 0.55;
-const GENERIC_TEMPLATE = `# {{Title}}
-
-## Summary
-
-## Details
-
-## Related`;
 
 export type Memory = {
 	name: string;
 	description: string;
 	content: string;
-	/** Description embedding — indexed in the KD tree, used for merge/ANN candidate lookup */
 	embedding: number[];
-	/** Title-only embedding, weighted heaviest during recall ranking */
 	titleEmbedding?: number[];
-	/** Chunked body embeddings, best-chunk match used during recall ranking */
 	bodyEmbeddings?: number[][];
 	links: string[];
 	backlinks: string[];
@@ -33,7 +23,6 @@ export type Memory = {
 type MemoryRef = {
 	name: string;
 	description: string;
-	/** Cosine distance from the query, present when returned from a search */
 	distance?: number;
 }
 
@@ -42,9 +31,17 @@ type FactBucket = {
 	facts: string[];
 }
 
+type MemoryTask = {
+	/** Exact node name / new persistent entity path this task belongs to, or '' for a personal task with no entity (goes to the journal) */
+	subject: string;
+	task: string;
+	done: boolean;
+}
+
 type FactAgentResult = {
 	buckets: FactBucket[];
 	journal: string;
+	tasks: MemoryTask[];
 }
 
 function dedupeFacts(facts: string[]): string[] {
@@ -75,7 +72,6 @@ function cosineSearch(query: number[], memories: Memory[], limit: number): Memor
 		.slice(0, limit);
 }
 
-/** Re-embed a node's title / description / body fields. Description embedding stays the KD-tree index key. */
 async function embedMemoryFields(node: Memory, llm: any): Promise<void> {
 	const body = stripHeader(node.content);
 	const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name);
@@ -90,9 +86,14 @@ export function stripHeader(content: string): string {
 	return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
 }
 
+/** True if a task has no persistent entity of its own and belongs in the journal instead. */
+function isPersonalTask(t: MemoryTask): boolean {
+	const s = (t.subject ?? '').trim().toLowerCase();
+	return !s || s === 'journal' || s.startsWith('journal/');
+}
+
 export class MemoryCache {
 	private tree!: KDTree<MemoryRef>;
-	/** Tracks which memories are currently indexed in the tree, keyed by name -> embedding reference */
 	private indexed = new Map<string, number[]>();
 	public memories: Memory[];
 	public nodes: MemoryNode[] = [];
@@ -105,7 +106,6 @@ export class MemoryCache {
 		this.rebuild();
 	}
 
-	/** Incrementally sync the KD tree against `this.memories` instead of rebuilding from scratch */
 	private syncTree(): void {
 		const current = new Set(this.memories.map(m => m.name));
 
@@ -187,7 +187,6 @@ class MemoryAccessor {
 		return nodes.filter(n => n.missing).map(n => n.name);
 	}
 
-	/** Cache path uses the KD tree's knn(); raw-array path (no cache available) falls back to a linear cosine scan */
 	search(vector: number[], limit: number): MemoryRef[] {
 		return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit);
 	}
@@ -248,10 +247,10 @@ export class MemoryManager {
 			name: 'memory_recall',
 			description: 'Read the full content of a memory document',
 			args: {
-				name: {type: 'string', description: 'Exact memory name', required: true},
+				name: {type: 'string', description: 'Exact memory name', required: true}
 			},
 			fn: (args: any) => {
-				const mem = this.access(memories).find(args.name);
+				const mem = new MemoryAccessor(memories).find(args.name);
 				if(!mem) return 'Document not found';
 				this.touch(mem.name);
 				return mem.content;
@@ -266,7 +265,7 @@ export class MemoryManager {
 				limit: {type: 'number', description: 'Number of memories to return', default: 1},
 			},
 			fn: async ({query, limit}) => {
-				const mem = await this.recollect(query, memories, limit)
+				const mem = await this.recollect(query, memories, limit);
 				return mem.map(m => `Memory: ${m.name}
 Description: ${m.description}
 Links: ${[...m.links, ...m.backlinks].join(', ')}
@@ -285,12 +284,11 @@ ${m.content}
 		return raw ? {memory: <Memory[] | MemoryCache>m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m};
 	}
 
-	private access(memories: Memory[] | MemoryCache): MemoryAccessor {
-		return new MemoryAccessor(memories);
-	}
-
 	private stage(node: Memory, block: string): void {
-		this.ensureDoc(node);
+		if(!node.content) {
+			const title = node.name.split('/').pop() ?? node.name;
+			node.content = this.touchHeader(node, `# ${title}\n`);
+		}
 		const body = stripHeader(node.content);
 		const idx = body.indexOf(PENDING_HEADING);
 		const newBody = idx === -1
@@ -299,38 +297,17 @@ ${m.content}
 		node.content = this.touchHeader(node, newBody);
 	}
 
-	private ensureDoc(node: Memory): void {
+	private resolveSubject(subject: string, store: MemoryAccessor): string {
-		if (node.content) return;
+		function normalize(name: string): string {
-		const title = node.name.split('/').pop() ?? node.name;
-		node.content = this.touchHeader(node, `# ${title}\n`);
-	}
-
-	private sanitizeDescription(text: string): string {
-		return (text ?? '').replace(/\s+/g, ' ').trim().slice(0, 240);
-	}
-
-	private relink(memories: Memory[], from: string, to: string): void {
-		const pattern = new RegExp(`\\[\\[${escapeRegex(from)}\\]\\]`, 'g');
-		for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`);
-	}
-
-	private normalizeLeaf(name: string): string {
 			return name.trim().toLowerCase().replace(/\s+/g, ' ');
 		}
 
-	/**
-	 * Resolve a fact-agent proposed subject to an existing node when it's an alias/rename of one.
-	 * Exact match is checked first (cheap, and covers the common case since node names are
-	 * already normalized at creation time). Only falls through to fuzzy alias matching against
-	 * same-root candidates when there's no existing hit — i.e. only on likely-new-doc creation.
-	 */
-	private resolveSubject(subject: string, store: MemoryAccessor): string {
 		const trimmed = subject.trim();
 		const exact = store.find(trimmed);
 		if(exact) return exact.name;
 
-		const normalized = this.normalizeLeaf(trimmed);
+		const normalized = normalize(trimmed);
-		const caseInsensitive = store.list.find(m => this.normalizeLeaf(m.name) === normalized);
+		const caseInsensitive = store.list.find(m => normalize(m.name) === normalized);
 		if(caseInsensitive) return caseInsensitive.name;
 
 		const root = trimmed.split('/')[0];
@@ -338,7 +315,6 @@ ${m.content}
 		const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
 		if(!candidates.length) return trimmed;
 
-		// fuzzyMatch requires >=2 terms; pad with an empty string when there's only one candidate
 		const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
 		const probe = leaves.length > 1 ? leaves : [...leaves, ''];
 		const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe);
@@ -353,36 +329,70 @@ ${m.content}
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
-			system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce:
+			system: `Turn this conversation into a persistent memory file by extracting information into organized bullet points
 
-1. Journal recap (single paragraph)
+Think of this like an Obsidian vault with a clear division of responsibility:
-   - "Captains Log" style record keeping
+- The JOURNAL is a timeline. It answers "what happened, and when" and is the only place with a sense of time.
-   - What was discussed/worked on, decisions, user's events/state/mood, general context
+- ENTITY DOSSIERS are a wiki. They answer "what is currently true about this subject", with no sense of time — only current state.
-   - Leave empty only for trivial/empty exchanges/small talk
+- Never blur the two: a one-off event, conversation, or debugging session is a journal entry, not an entity, even if it's detailed.
 
-2. Fact buckets
+1. Journal Log
-   - ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation
+- A chronological, skimmable log of what actually happened: real discussions, decisions made, progress on projects, problems worked through
-   - NEVER extract greetings, pleasantries, or anything the assistant itself said
+- This is NOT a transcript, and it is NOT a step-by-step record, its a compressed log of notable events & developments
-   - Extract the final/end state, not deltas
+- One line per development is usually enough: what was worked on and the outcome, not the blow-by-blow of how
+- Skip small talk and trivial exchanges entirely. Skip anything that's purely a todo item (goes in Todo Tasks) or a durable fact about a subject (goes in Entity Dossiers)
 
-Path assignment rules:
+2. Todo Tasks
-   - Reuse existing node names whenever possible, including when the subject is an alias/nickname of an existing node (e.g. "Rob" referring to an existing "People/Robert")
+- Extract concrete tasks the user says need to be done, should be done, or were completed
-   - Documents should be grouped and named by the root subject
+- Return the task text and whether it is still todo or is done
-     - Person → People/Name
+- A completed task should be marked done, not recreated as a new todo
-     - Project → Projects/Name
+- Only extract actionable tasks, not general goals or observations
-     - Concept → Concepts/Name
+- Assign each task a subject:
-   - A bug report, its investigation, should be nested and attached to the same root subject node
+    - If the task belongs to a persistent entity (a project, a class, etc.), use that entity's exact node name, or a new entity path if it doesn't exist yet
-   - Tickets/one-off tasks → file under the project/name/component they belong to
+    - If it's a personal/life task with no entity of its own (reach out to someone, reply to an email, pay a bill, etc.), leave subject as an empty string — it belongs in the journal, not a new document
-   - Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\`
+
+3. Entity Dossiers
+- Detailed dossiers with all information regarding a subject
+- Record the final/end state, not intermediate changes
+- Ignore assistant claims, guesses, greetings, or temporary details
+- NEVER create a document for something that's only meaningful as a point in time — a single conversation, a one-off decision, a debugging session, a date. That's a journal entry, not an entity
+- identify its HOME ENTITY:
+    - The HOME ENTITY name should always be a [abstract|pro]noun
+    - The grammatical subject/owner of the fact is the strongest clue
+    - Always preference an existing entity over creating a new one
+    - New child entities are appropriate only when they are themselves distinct persistent entities
+    - A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment
+    - Put project facts under the project they belong to, person facts under the person, etc
+
+Example Entity Naming Convention:
+- Projects/[Name]
+- People/[Name]
+- History/[Name]
+- Science/[Name]
+- [Subject]/[Name]
+- Class/[Name]/[Chapter]
+
+Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships
+Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent
 
 Available nodes:
 ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
 ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			schema: {
 				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
-				buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
+				tasks: {
+					type: 'array', description: 'Concrete tasks mentioned or completed in the conversation.', required: false, items: {
 						type: 'object', items: {
-							subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
+							subject: {type: 'string', description: 'Exact node name / new persistent entity path this task belongs to, or an empty string if this is a personal task with no entity of its own (those go in the journal)', required: true},
+							task: {type: 'string', description: 'Concise actionable task', required: true},
+							done: {type: 'boolean', description: 'Whether the task is completed', required: true},
+						},
+					}
+				},
+				buckets: {
+					type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
+						type: 'object', items: {
+							subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true},
 							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
 						},
 					},
@@ -401,10 +411,11 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return {
 			buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
 			journal: (response.journal ?? '').trim(),
+			tasks: response.tasks ?? [],
 		};
 	}
 
-	private getWeekMonday(date: Date = new Date()): string {
+	private getWeekStart(date: Date = new Date()): string {
 		const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
 		const day = d.getUTCDay();
 		const diff = day === 0 ? -6 : 1 - day;
@@ -412,38 +423,86 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return d.toISOString().slice(0, 10);
 	}
 
+	private journalDescription(journalName?: string): string {
+		const start = journalName?.split('/').pop() || this.getWeekStart();
+		const d = new Date(`${start}T00:00:00Z`);
+		d.setUTCDate(d.getUTCDate() + 6);
+		const end = d.toISOString().slice(0, 10);
+		return `Log from ${start} - ${end}`;
+	}
+
+	private getIncompleteTodos(content: string): string[] {
+		const body = stripHeader(content);
+		const match = body.match(/## Todo list\n([\s\S]*?)(?=\n## |$)/i);
+		if(!match) return [];
+		return match[1].split('\n')
+			.map(line => line.match(/^\s*-\s*\[([ xX])\]\s+(.+?)\s*$/))
+			.filter((m): m is RegExpMatchArray => !!m && m[1].toLowerCase() !== 'x')
+			.map(m => m[2].trim());
+	}
+
 	private listNodes(memories: Memory[]): MemoryRef[] {
 		return memories.map(m => ({name: m.name, description: m.description}));
 	}
 
-	/** Finds the closest merge candidate via the KD tree's knn() instead of a manual O(n) cosine scan */
+	private async mergeAgent(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory | null> {
-	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
+		function factSimilarity(a: Memory, b: Memory): number {
+			if(!a.bodyEmbeddings?.length || !b.bodyEmbeddings?.length) return 0;
+			let best = 0;
+			for(const av of a.bodyEmbeddings) {
+				for(const bv of b.bodyEmbeddings) best = Math.max(best, 1 - cosineDistance(av, bv));
+			}
+			return best;
+		}
+
 		if(!node.embedding?.length || node.name.startsWith('Journal/')) return null;
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
+		const candidates = store.list
+			.filter(m => m.name !== node.name && !m.name.startsWith('Journal/'))
+			.filter(m => factSimilarity(node, m) >= FACT_SIMILARITY_THRESHOLD);
 
-		const candidate = store.search(node.embedding, 5)
+		if(!candidates.length) return null;
-			.find(r => r.name !== node.name && !r.name.startsWith('Journal/') && r.distance !== undefined && r.distance <= threshold);
+		const closest = candidates.sort((a, b) => factSimilarity(node, b) - factSimilarity(node, a))[0];
-		if (!candidate) return null;
+		const result = await this.llm.ask('', {
-		const closest = store.find(candidate.name);
+			model: options.model,
-		if (!closest) return null;
+			temperature: 0.3,
+			schema: {
+				aContent: {type: 'string', description: 'Updated document A body in markdown, without frontmatter.', required: true},
+				bContent: {type: 'string', description: 'Updated document B body in markdown, without frontmatter.', required: true},
+			},
+			system: `Maintain these two persistent knowledge-base documents like a wiki.
 
-		const result = await this.mergeAgent(node, closest, options);
+Do NOT merge, rename, or delete either document. Both represent entities that should remain independently addressable.
-		const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []};
-		merged.content = this.touchHeader(merged, result.content);
-		await embedMemoryFields(merged, this.llm);
 
-		this.relink(store.list, node.name, merged.name);
+The documents were selected because their facts may overlap. Your job is to reconcile duplicated information and connect the documents:
-		this.relink(store.list, closest.name, merged.name);
+- Decide which document is the HOME for each duplicated fact.
+- Keep the authoritative copy in that home document.
+- In the other document, replace the information with a short preamble and [[WikiLink]] to the home entity explaining the relationship.
+- If the documents are distinct entities but merely related, keep their distinct facts and add useful [[WikiLinks]] between them.
+- Do not delete useful entity-specific facts just because they are similar.
+- Do not invent relationships or facts.
+- Preserve useful history, technical specifics, structure, and existing [[WikiLinks]].
+- Most current truth wins when facts conflict.
+- Keep both documents concise and information-dense.
+- No frontmatter, preamble, filler, or AI commentary.
 
-		this.queues.get(closest.name)?.request?.abort?.();
+Document A ("${node.name}"):
-		this.queues.delete(closest.name);
+\`\`\`markdown
+${stripHeader(node.content)}
+\`\`\`
 
-		store.forget(node.name);
+Document B ("${closest.name}"):
-		store.forget(closest.name);
+\`\`\`markdown
-		store.list.push(merged);
+${stripHeader(closest.content)}
-		store.commit();
+\`\`\``,
-
+		});
-		return merged;
+		const a = store.find(node.name);
+		const b = store.find(closest.name);
+		if(!a || !b || !result?.aContent || !result?.bContent) return null;
+		a.content = this.touchHeader(a, result.aContent);
+		b.content = this.touchHeader(b, result.bContent);
+		await Promise.all([embedMemoryFields(a, this.llm), embedMemoryFields(b, this.llm)]);
+		return a;
 	}
 
 	private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
@@ -457,19 +516,19 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 
 		const entry = {dirty: false, request: null, task: Promise.resolve()};
 		this.queues.set(key, entry);
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		entry.task = (async () => {
-			let current = node, merged = false;
+			let current = node;
 			try {
 				do {
 					entry.dirty = false;
 					await this.docAgent(current, store.list, options, entry);
-					this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
+					this.mergeLock = this.mergeLock.then(() => this.mergeAgent(current, memories, options));
 					const result = await this.mergeLock;
-					if (result) { current = result; merged = true; }
+					if(result) current = result;
 				} while(entry.dirty);
 			} finally {
-				store.commit(merged ? undefined : [node]);
+				store.commit([node]);
 				this.queues.delete(key);
 			}
 		})();
@@ -479,6 +538,40 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 	private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
 		if(!memories.includes(node)) return;
 		const currentBody = stripHeader(node.content);
+		const journal = node.name.startsWith('Journal/');
+		const system = (journal
+			? `You maintain one persistent journal document
+
+Rewrite the ENTIRE journal, folding "## Pending" into the existing content removing the heading
+
+Journal design:
+- Preserve the chronological daily log
+- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
+- Group information by day under a date heading
+- Keep journal entries high level and concise: what was worked on and the outcome, not a step-by-step record of how — that detail lives in conversation history, not here
+- Use [[WikiLinks]] for persistent entities; don't turn ordinary journal events into entities
+- No frontmatter, preamble, filler, or AI commentary`
+			: `You maintain one persistent knowledge-base entity document
+
+Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished.
+
+Document design:
+- The document represents one persistent entity. Keep information about that entity together and organized into sections
+- Merge any pending information in, newest fact wins conflicts; remove redundant content
+- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
+- Let the structure fit the entity; there is NO fixed template
+- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
+- Keep the document concise and information-dense without removing useful technical specifics
+- Current truth wins when facts conflict. Preserve older conflict as context, only when it adds useful meaning
+- No frontmatter, preamble, filler, or AI commentary`) + `
+
+Available nodes to link to:
+${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
+
+Current document:
+\`\`\`markdown
+${currentBody}
+\`\`\``;
 		let update;
 		try {
 			for(let i = 0; i < 2 && !update?.content; i++) {
@@ -489,30 +582,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
-					system: `You are a knowledge base editor maintaining one Obsidian-style document.
+					system,
-
-If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
-
-Use this loose structure, adapting headings to what the content needs:
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
-
-Rules:
-- Contradictions: "## Pending" holds the newest information — bias toward it. Fold it in as the standing fact and drop the outdated statement, unless the old context adds meaningful nuance (e.g. "previously X, now Y"). This document should read as a source of truth, not an audit log
-- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
-- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
-- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
-- Keep concise, factual, human-readable
-- NO frontmatter, filler, preamble, or AI commentary
-
-Available nodes to link to (don't duplicate their content):
-${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
-
-Current document:
-\`\`\`markdown
-${currentBody}
-\`\`\``,
 				});
 				entry.request = request;
 				update = await request;
@@ -525,43 +595,11 @@ ${currentBody}
 		}
 
 		if(!update?.content) return;
-		node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
+		node.description = node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.name !== 'People/User' ? update.description.replaceAll(/[\n:]/g, '') : 'All information about the current user';
 		node.content = this.touchHeader(node, update.content);
 		await embedMemoryFields(node, this.llm);
 	}
 
-	private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> {
-		const modifiedOf = (m: Memory) => this.parseFrontmatter(m.content).fm.get('modified') || 'unknown';
-
-		return this.llm.ask('', {
-			model: options.model,
-			temperature: 0.3,
-			schema: {
-				name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true},
-				description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
-				content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
-			},
-			system: `You are a knowledge base editor merging two overlapping Obsidian documents into one.
-
-Structure loosely:
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
-
-Combine both documents, resolve duplication. On contradictions, bias toward whichever document was modified more recently; drop the outdated statement unless the old context adds meaningful nuance.
-
-Document A ("${a.name}", last modified ${modifiedOf(a)}):
-\`\`\`markdown
-${stripHeader(a.content)}
-\`\`\`
-
-Document B ("${b.name}", last modified ${modifiedOf(b)}):
-\`\`\`markdown
-${stripHeader(b.content)}
-\`\`\``,
-		});
-	}
-
 	private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
 		const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
 		if(!match) return {fm: new Map(), body: content};
@@ -572,21 +610,16 @@ ${stripHeader(b.content)}
 			const key = line.slice(0, i).trim();
 			const raw = line.slice(i + 1).trim();
 			let value = raw;
-			try { value = JSON.parse(raw); } catch { /* legacy unquoted value, keep raw */ }
+			try { value = JSON.parse(raw); } catch { }
 			fm.set(key, value);
 		}
 		return {fm, body: match[2]};
 	}
 
-	/**
-	 * Writes the code-owned frontmatter block. `body` is passed through stripHeader() first so a
-	 * model that ignores instructions and hallucinates its own `---` block can never corrupt or
-	 * duplicate the real frontmatter — the LLM only ever gets to influence the body.
-	 */
 	private touchHeader(node: Memory, body: string): string {
 		const {fm} = this.parseFrontmatter(node.content);
 		fm.set('name', node.name);
-		fm.set('description', node.description || '');
+		fm.set('description', (node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.description) || 'Persistent memory document');
 		fm.set('modified', new Date().toISOString());
 		return this.writeFrontmatter(fm, stripHeader(body));
 	}
@@ -608,11 +641,11 @@ ${stripHeader(b.content)}
 	}
 
 	forget(name: string, memories: Memory[] | MemoryCache): boolean {
-		return this.access(memories).forget(name);
+		return new MemoryAccessor(memories).forget(name);
 	}
 
-	/** Ranks a candidate pool by weighted title/description/body similarity against the query embedding */
+	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
-	private rankByFields(query: number[], candidates: Memory[], limit: number): Memory[] {
+		function rank(query: number[], candidates: Memory[], limit: number): Memory[] {
 			const scored = candidates.map(m => {
 				const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0;
 				const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0;
@@ -624,19 +657,16 @@ ${stripHeader(b.content)}
 			return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory);
 		}
 
-	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
+		const store = new MemoryAccessor(memories);
-		const store = this.access(memories);
 		if(!store.list.length) return [];
-
 		await store.backfillEmbeddings(this.llm);
 
 		const [e] = await this.llm.embedding(query);
 		if(!e) return [];
 
-		// Description embedding is the cheap ANN index key; pull a wider pool then re-rank by field weight
 		const pool = store.search(e.embedding, Math.max(limit * 3, limit));
 		const poolMemories = pool.map(r => store.find(r.name)).filter((m): m is Memory => !!m);
-		const ranked = this.rankByFields(e.embedding, poolMemories, limit);
+		const ranked = rank(e.embedding, poolMemories, limit);
 		const found = new Set<string>(ranked.map(m => m.name));
 
 		if(graphDepth > 0) {
@@ -672,29 +702,69 @@ ${stripHeader(b.content)}
 		const pending = {role: 'tool', name: 'memory_process', id: uid, content: conversation} as unknown as LLMMessage;
 		history.push(pending);
 
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
-		const {buckets, journal} = await this.factAgent(conversation, store, options);
+		const {buckets, journal, tasks} = await this.factAgent(conversation, store, options);
 		const touched: Memory[] = [];
 
-		if (journal) {
+		const personalTasks = tasks.filter(isPersonalTask);
-			const journalName = `Journal/${this.getWeekMonday()}`;
+		const entityTasks = tasks.filter(t => !isPersonalTask(t));
+
+		if(journal || personalTasks.length) {
+			const journalName = `Journal/${this.getWeekStart()}`;
 			let jnode = store.find(journalName);
+			const isNew = !jnode;
 			if(!jnode) {
-				jnode = {name: journalName, description: '', content: '', embedding: [], links: [], backlinks: []};
+				jnode = {
+					name: journalName,
+					description: this.journalDescription(),
+					content: '',
+					embedding: [],
+					links: [],
+					backlinks: [],
+				};
 				store.list.push(jnode);
 			}
-			this.stage(jnode, `### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+
+			const blocks: string[] = [];
+			if(journal) blocks.push(`### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+			if(isNew) {
+				const previousDate = new Date(`${this.getWeekStart()}T00:00:00Z`);
+				previousDate.setUTCDate(previousDate.getUTCDate() - 7);
+				const previous = store.find(`Journal/${previousDate.toISOString().slice(0, 10)}`);
+				if(previous) {
+					const todos = this.getIncompleteTodos(previous.content);
+					if(todos.length) blocks.push(`${TODO_HEADING}\n${todos.map(task => `- [ ] ${task}`).join('\n')}`);
+				}
+			}
+			if(personalTasks.length) blocks.push(`${TODO_HEADING}\n${personalTasks.map(task => `- [${task.done ? 'x' : ' '}] ${task.task}`).join('\n')}`);
+			if(blocks.length) this.stage(jnode, blocks.join('\n\n'));
 			touched.push(jnode);
 		}
 
+		const entityStaging = new Map<string, {facts: string[], tasks: MemoryTask[]}>();
 		for(const {subject, facts} of buckets) {
 			const resolved = this.resolveSubject(subject, store);
+			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
+			entry.facts.push(...facts);
+			entityStaging.set(resolved, entry);
+		}
+		for(const task of entityTasks) {
+			const resolved = this.resolveSubject(task.subject, store);
+			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
+			entry.tasks.push(task);
+			entityStaging.set(resolved, entry);
+		}
+
+		for(const [resolved, {facts, tasks: subjectTasks}] of entityStaging) {
 			let node = store.find(resolved);
 			if(!node) {
-				node = {name: resolved, description: '', content: '', embedding: [], links: [], backlinks: []};
+				node = {name: resolved, description: 'Persistent memory document', content: '', embedding: [], links: [], backlinks: []};
 				store.list.push(node);
 			}
-			this.stage(node, facts.map(f => `- ${f}`).join('\n'));
+			const blocks: string[] = [];
+			if(facts.length) blocks.push(facts.map(f => `- ${f}`).join('\n'));
+			if(subjectTasks.length) blocks.push(`${TODO_HEADING}\n${subjectTasks.map(t => `- [${t.done ? 'x' : ' '}] ${t.task}`).join('\n')}`);
+			if(blocks.length) this.stage(node, blocks.join('\n\n'));
 			touched.push(node);
 		}
 
@@ -716,7 +786,7 @@ ${stripHeader(b.content)}
 	}
 
 	async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING));
 		await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
 		store.commit();

src/open-ai.ts

@@ -36,21 +36,49 @@ export class OpenAi extends LLMProvider {
 	private toWire(history: LLMMessage[], system?: string): any[] {
 		const wire: any[] = [];
 		if(system) wire.push({role: 'system', content: system});
-		for(const h of history) {
+
-			if(h.role === 'tool') {
+		for(let i = 0; i < history.length; i++) {
+			const h = history[i];
+
+			if(h.role !== 'tool') {
+				wire.push({role: h.role, content: this.toWireContent(h.content)});
+				continue;
+			}
+
+			const calls: any[] = [];
+			const results: any[] = [];
+
+			while(i < history.length && history[i].role === 'tool') {
+				const tool: any = history[i];
+
+				calls.push({
+					id: tool.id,
+					type: 'function',
+					function: {
+						name: tool.name,
+						arguments: JSON.stringify(tool.args || {})
+					}
+				});
+
+				results.push({
+					role: 'tool',
+					tool_call_id: tool.id,
+					content: tool.error || tool.content || ''
+				});
+
+				i++;
+			}
+
 			wire.push({
 				role: 'assistant',
 				content: null,
-					tool_calls: [{id: h.id, type: 'function', function: {name: h.name, arguments: JSON.stringify(h.args)}}],
+				tool_calls: calls
-				}, {
-					role: 'tool',
-					tool_call_id: h.id,
-					content: h.error || h.content || '',
 			});
-			} else {
+
-				wire.push({role: h.role, content: this.toWireContent(h.content)});
+			wire.push(...results);
-			}
+			i--;
 		}
+
 		return wire;
 	}
 
@@ -60,13 +88,12 @@ export class OpenAi extends LLMProvider {
 			if(!options.history) options.history = [];
 			const history = options.history;
 			if(message) history.push({role: 'user', content: message, timestamp: Date.now()});
-
 			const tools = options.tools || this.ai.options.llm?.tools || [];
 			const requestParams: any = {
 				model: options.model || this.model,
 				stream: !!options.stream,
-				max_completion_tokens: options.maxTokens || this.ai.options.llm?.maxTokens || undefined,
+				max_completion_tokens: options.maxTokens ?? this.ai.options.llm?.maxTokens,
-				temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
+				temperature: options.temperature ?? this.ai.options.llm?.temperature,
 				tools: tools.map(t => ({
 					type: 'function',
 					function: {
@@ -74,8 +101,12 @@ export class OpenAi extends LLMProvider {
 						description: t.description,
 						parameters: {
 							type: 'object',
-							properties: t.args ? objectMap(t.args, (key, value) => ({...value, required: undefined})) : {},
+							properties: t.args
-							required: t.args ? Object.entries(t.args).filter(t => t[1].required).map(t => t[0]) : []
+								? objectMap(t.args, (key, value) => ({...value, required: undefined}))
+								: {},
+							required: t.args
+								? Object.entries(t.args).filter(t => t[1].required).map(t => t[0])
+								: []
 						}
 					}
 				}))
@@ -83,60 +114,106 @@ export class OpenAi extends LLMProvider {
 
 			if(options.schema) {
 				const schema = convertSchema(options.schema);
-				requestParams.response_format = {type: 'json_schema', json_schema: {name: 'response', strict: true, schema}};
+				requestParams.response_format = {
+					type: 'json_schema',
+					json_schema: {name: 'response', strict: true, schema}
+				};
 			}
 			if(options.stream) requestParams.stream_options = {include_usage: true};
 
 			try {
 				let terminal = false;
+				let iteration = 0;
+
 				do {
+					iteration++;
 					requestParams.messages = this.toWire(history.filter(h => h.role !== 'system'), options.system);
 
 					const callStart = Date.now();
-					const resp: any = await this.tokenPool.run(token => this.getClient(token).chat.completions.create(requestParams)).catch(err => {
+					const resp: any = await this.tokenPool.run(token =>
+						this.getClient(token).chat.completions.create(requestParams)
+					).catch(err => {
 						err.message += `\n\nMessages:\n${JSON.stringify(requestParams.messages, null, 2)}`;
 						throw err;
 					});
 
-					let usage: any, msg: any = {content: '', tool_calls: []};
+					let usage: any;
+					let finishReason: string | undefined;
+					let msg: any = {content: '', tool_calls: []};
+					let streamedChars = 0;
+
 					if(options.stream) {
+						let streamCompleted = false;
+						try {
 							for await (const chunk of resp) {
 								if(controller.signal.aborted) break;
 								if(chunk.usage) usage = chunk.usage;
-							if(chunk.choices[0]?.delta?.content) {
+
-								msg.content += chunk.choices[0].delta.content;
+								const choice = chunk.choices?.[0];
-								options.stream({text: chunk.choices[0].delta.content});
+								if(choice?.finish_reason) finishReason = choice.finish_reason;
+
+								if(choice?.delta?.content) {
+									msg.content += choice.delta.content;
+									streamedChars += choice.delta.content.length;
+									options.stream({text: choice.delta.content});
 								}
-							if(chunk.choices[0]?.delta?.tool_calls) {
+
-								for(const deltaTC of chunk.choices[0].delta.tool_calls) {
+								if(choice?.delta?.tool_calls) {
-									const existing = msg.tool_calls.find((tc: any) => tc.index === deltaTC.index);
+									for(const deltaTC of choice.delta.tool_calls) {
-									if(existing) {
+										const index = deltaTC.index ?? msg.tool_calls.length;
+										let existing = msg.tool_calls.find((tc: any) => tc.index === index);
+
+										if(!existing) {
+											existing = {index, id: '', function: {name: '', arguments: ''}};
+											msg.tool_calls.push(existing);
+										}
+
 										if(deltaTC.id) existing.id = deltaTC.id;
 										if(deltaTC.function?.name) existing.function.name = deltaTC.function.name;
 										if(deltaTC.function?.arguments) existing.function.arguments += deltaTC.function.arguments;
-									} else {
-										msg.tool_calls.push({
-											index: deltaTC.index,
-											id: deltaTC.id || '',
-											function: {name: deltaTC.function?.name || '', arguments: deltaTC.function?.arguments || ''}
-										});
 									}
 								}
 							}
+
+							streamCompleted = true;
+						} catch(err) {
+							if(!controller.signal.aborted) throw err;
 						}
+
+						if(streamCompleted && !finishReason) finishReason = msg.tool_calls.length ? 'tool_calls' : 'stop';
 					} else {
 						usage = resp.usage;
+						finishReason = resp.choices[0].finish_reason;
 						msg = resp.choices[0].message;
 					}
+
 					const duration = Date.now() - callStart;
 					const tps = usage?.completion_tokens && duration > 0 ? usage.completion_tokens / (duration / 1000) : 0;
 
+					if(finishReason === 'length' && !controller.signal.aborted) {
+						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
+						throw new Error(`[OpenAI] Response hit token limit before completing`);
+					}
+
+					if(!finishReason && !controller.signal.aborted) {
+						throw new Error('[OpenAI] Completion ended without a usable response');
+					}
+
 					const toolCalls = msg.tool_calls || [];
+
 					if(toolCalls.length && !controller.signal.aborted) {
 						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
 
 						const entries = toolCalls.map((tc: any) => {
-							const entry: any = {role: 'tool', id: tc.id, name: tc.function.name, args: JSONAttemptParse(tc.function.arguments, {}), content: undefined, timestamp: Date.now()};
+							const entry: any = {
+								role: 'tool',
+								id: tc.id,
+								name: tc.function.name,
+								args: JSONAttemptParse(tc.function.arguments, {}),
+								content: undefined,
+								timestamp: Date.now()
+							};
+
 							history.push(entry);
 							return {tc, entry};
 						});
@@ -144,12 +221,13 @@ export class OpenAi extends LLMProvider {
 						await Promise.all(entries.map(async ({tc, entry}: any) => {
 							const tool = tools.find(findByProp('name', tc.function.name));
 							if(options.stream) options.stream({tool: tc.function.name});
-							if(!tool) { entry.error = 'Tool not found'; return; }
+							if(!tool) return entry.error = 'Tool not found';
 							try {
 								const toolStream = options.stream && ((chunk: any) => {
-									if(chunk.done) { terminal = true; return; }
+									if(chunk.done) return;
 									options.stream!(chunk);
 								});
+
 								const result = await tool.fn(entry.args, toolStream, this.ai, tc.id);
 								entry.content = typeof result === 'object' ? JSONSanitize(result) : result;
 							} catch(err: any) {
@@ -164,7 +242,6 @@ export class OpenAi extends LLMProvider {
 				} while(!terminal && !controller.signal.aborted);
 
 				if(options.stream) options.stream({done: true});
-
 				const turnStart = history.map(h => h.role).lastIndexOf('user');
 				const finalContent = history.slice(turnStart + 1).reduce((str, h) => h.role === 'assistant' ? str + (h.content || '') : str, '').trim();
 				res(options.schema ? JSONAttemptParse(finalContent, finalContent) : finalContent);