Recursive agents update

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.7",
+	"version": "1.7.0",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/llm.ts

@@ -19,6 +19,13 @@ const PDF_OCR_PAGE_THRESHOLD = 12; // above this many pages, OCR scanned pages i
 export type AnthropicConfig = {proto: 'anthropic', token: string | string[]};
 export type OpenAiConfig = {proto: 'openai', host?: string, token: string | string[]};
 
+export type AgentRef = {
+	name: string;
+	description?: string;
+	delegate?: boolean;
+	fn: () => Agent | null | Promise<Agent | null>;
+}
+
 export type Agent = {
 	name: string;
 	description?: string;
@@ -29,7 +36,7 @@ export type Agent = {
 	skills?: Skill[] | null;
 	tools?: AiTool[] | null;
 	mcp?: McpServer[] | null;
-	agents?: string[] | null;
+	agents?: AgentRef[] | null;
 }
 
 export type LLMFile = {
@@ -106,8 +113,8 @@ export type LLMRequest = {
 	skills?: Skill[];
 	/** MCP servers to connect and expose as tools */
 	mcp?: McpServer[];
-	/** Subagents exposed as delegatable/wrapped tools */
-	agents?: Agent[];
+	/** Subagents exposed as delegatable/wrapped tools, resolved lazily via their `fn` */
+	agents?: AgentRef[];
 	/** Attach files to request */
 	files?: LLMFile[];
 	/** @internal recursion guard for nested agent delegation */
@@ -265,22 +272,21 @@ class LLM {
 		};
 	}
 
-	private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: ((keep?: boolean) => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
-		return agents.map(a => {
-			const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`;
+	private setupAgent(stubs: AgentRef[] = [], history: LLMMessage[], aborts: ((keep?: boolean) => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
+		return stubs.map(stub => {
+			const toolName = `${stub.delegate ? '' : 'sub'}agent_${snakeCase(stub.name)}`;
 			return {
 				name: toolName,
-				description: `${a.delegate ? 'Delegate to ' : ''}Subagent: ${a.description || a.name}`,
+				description: `${stub.delegate ? 'Delegate to ' : ''}Subagent: ${stub.description || stub.name}`,
 				args: clean<any>({
-					context: !a.delegate ? {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true} : undefined,
+					context: !stub.delegate ? {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true} : undefined,
 					instructions: {type: 'string', description: 'Detailed instructions for subagent to complete', required: true},
 				}),
 				fn: async (args: any, stream: any, ai: any, id?: string) => {
 					if(depth >= MAX_AGENT_DEPTH) return 'Max agent delegation depth exceeded';
 
-					const nested = (a.agents || [])
-						.map(name => allAgents.find(x => x.name === name))
-						.filter((x): x is Agent => !!x && x.name !== a.name);
+					const a = await stub.fn();
+					if(!a) return `Agent "${stub.name}" could not be resolved`;
 
 					const q = a.delegate ? '' : `${args.instructions}${args.context ? `\n\n<context>${args.context}</context>` : ''}`;
 
@@ -297,7 +303,7 @@ ${a.system}`,
 						mcp: a.mcp || undefined,
 						skills: a.skills || undefined,
 						tools: a.tools || undefined,
-						agents: nested,
+						agents: a.agents || [],
 						_agentDepth: depth + 1,
 					} as any);
 					aborts.push(request.abort);
@@ -451,7 +457,7 @@ ${a.system}`,
 			// Agents
 			const agents = options.agents || this.ai.options?.llm?.agents;
 			const delegateState: {resp: string | null} = {resp: null};
-			if(agents?.length) tools.push(...this.setupAgent(agents, agents, history, nestedAborts, options._agentDepth || 0, delegateState));
+			if(agents?.length) tools.push(...this.setupAgent(agents, history, nestedAborts, options._agentDepth || 0, delegateState));
 
 			// Memory
 			const mem = MemoryManager.normalize(options.memory);

src/memory.ts

@@ -2,29 +2,19 @@ import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts';
 import {LLMRequest, LLMMessage} from './llm.ts';
 import {AiTool} from './tools.ts';
 import {KDTree} from './kd-tree.ts';
-import {escapeRegex} from '@ztimson/utils';
 
-const MERGE_THRESHOLD = 0.12;
+const FACT_SIMILARITY_THRESHOLD = 0.62;
 const PENDING_HEADING = '## Pending';
+const TODO_HEADING = '## Todo list';
 const TREE_TOMBSTONE_LIMIT = 0.25;
 const ALIAS_MATCH_THRESHOLD = 0.55;
-const GENERIC_TEMPLATE = `# {{Title}}
-
-## Summary
-
-## Details
-
-## Related`;
 
 export type Memory = {
 	name: string;
 	description: string;
 	content: string;
-	/** Description embedding — indexed in the KD tree, used for merge/ANN candidate lookup */
 	embedding: number[];
-	/** Title-only embedding, weighted heaviest during recall ranking */
 	titleEmbedding?: number[];
-	/** Chunked body embeddings, best-chunk match used during recall ranking */
 	bodyEmbeddings?: number[][];
 	links: string[];
 	backlinks: string[];
@@ -33,7 +23,6 @@ export type Memory = {
 type MemoryRef = {
 	name: string;
 	description: string;
-	/** Cosine distance from the query, present when returned from a search */
 	distance?: number;
 }
 
@@ -42,23 +31,31 @@ type FactBucket = {
 	facts: string[];
 }
 
+type MemoryTask = {
+	/** Exact node name / new persistent entity path this task belongs to, or '' for a personal task with no entity (goes to the journal) */
+	subject: string;
+	task: string;
+	done: boolean;
+}
+
 type FactAgentResult = {
 	buckets: FactBucket[];
 	journal: string;
+	tasks: MemoryTask[];
 }
 
 function dedupeFacts(facts: string[]): string[] {
 	const seen = new Map<string, string>();
-	for (const f of facts) {
+	for(const f of facts) {
 		const clean = f.trim();
-		if (clean) seen.set(clean.toLowerCase(), clean);
+		if(clean) seen.set(clean.toLowerCase(), clean);
 	}
 	return [...seen.values()];
 }
 
 function cosineDistance(a: number[], b: number[]): number {
 	let dot = 0, normA = 0, normB = 0;
-	for (let i = 0; i < a.length; i++) {
+	for(let i = 0; i < a.length; i++) {
 		dot += a[i] * b[i];
 		normA += a[i] * a[i];
 		normB += b[i] * b[i];
@@ -75,14 +72,13 @@ function cosineSearch(query: number[], memories: Memory[], limit: number): Memor
 		.slice(0, limit);
 }
 
-/** Re-embed a node's title / description / body fields. Description embedding stays the KD-tree index key. */
 async function embedMemoryFields(node: Memory, llm: any): Promise<void> {
 	const body = stripHeader(node.content);
 	const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name);
 	const [descE] = await llm.embedding(node.description || '');
 	const bodyChunks = body ? await llm.embedding(body) : [];
-	if (titleE) node.titleEmbedding = titleE.embedding;
-	if (descE) node.embedding = descE.embedding;
+	if(titleE) node.titleEmbedding = titleE.embedding;
+	if(descE) node.embedding = descE.embedding;
 	node.bodyEmbeddings = bodyChunks.map((c: any) => c.embedding).filter(Boolean);
 }
 
@@ -90,9 +86,14 @@ export function stripHeader(content: string): string {
 	return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
 }
 
+/** True if a task has no persistent entity of its own and belongs in the journal instead. */
+function isPersonalTask(t: MemoryTask): boolean {
+	const s = (t.subject ?? '').trim().toLowerCase();
+	return !s || s === 'journal' || s.startsWith('journal/');
+}
+
 export class MemoryCache {
 	private tree!: KDTree<MemoryRef>;
-	/** Tracks which memories are currently indexed in the tree, keyed by name -> embedding reference */
 	private indexed = new Map<string, number[]>();
 	public memories: Memory[];
 	public nodes: MemoryNode[] = [];
@@ -105,31 +106,30 @@ export class MemoryCache {
 		this.rebuild();
 	}
 
-	/** Incrementally sync the KD tree against `this.memories` instead of rebuilding from scratch */
 	private syncTree(): void {
 		const current = new Set(this.memories.map(m => m.name));
 
-		for (const [name, emb] of [...this.indexed]) {
+		for(const [name, emb] of [...this.indexed]) {
 			const mem = this.memories.find(m => m.name === name);
-			if (!mem || !current.has(name) || mem.embedding !== emb) {
+			if(!mem || !current.has(name) || mem.embedding !== emb) {
 				this.tree.remove(p => p.name === name);
 				this.indexed.delete(name);
 			}
 		}
 
-		for (const mem of this.memories) {
-			if (!mem.embedding?.length || this.indexed.has(mem.name)) continue;
-			if (this.tree.dims === 0) this.tree = new KDTree<MemoryRef>(mem.embedding.length, 'cosine');
-			if (mem.embedding.length !== this.tree.dims) continue; // guard against embedding model/dim drift
+		for(const mem of this.memories) {
+			if(!mem.embedding?.length || this.indexed.has(mem.name)) continue;
+			if(this.tree.dims === 0) this.tree = new KDTree<MemoryRef>(mem.embedding.length, 'cosine');
+			if(mem.embedding.length !== this.tree.dims) continue; // guard against embedding model/dim drift
 			this.tree.insert({vector: mem.embedding, payload: {name: mem.name, description: mem.description}});
 			this.indexed.set(mem.name, mem.embedding);
 		}
 
-		if (this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance();
+		if(this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance();
 	}
 
 	search(query: number[], limit: number): MemoryRef[] {
-		if (!this.tree || this.tree.dims === 0) return [];
+		if(!this.tree || this.tree.dims === 0) return [];
 		return this.tree.knn(query, limit).map(r => ({...r.point.payload, distance: r.distance}));
 	}
 
@@ -140,14 +140,14 @@ export class MemoryCache {
 
 	update(memory: Memory): void {
 		const existing = this.memories.find(m => m.name === memory.name);
-		if (existing) Object.assign(existing, memory);
+		if(existing) Object.assign(existing, memory);
 		else this.memories.push(memory);
 		this.rebuild([existing ?? memory]);
 	}
 
 	remove(name: string): void {
 		const idx = this.memories.findIndex(m => m.name === name);
-		if (idx !== -1) {
+		if(idx !== -1) {
 			this.memories.splice(idx, 1);
 			this.rebuild();
 		}
@@ -175,7 +175,7 @@ class MemoryAccessor {
 	}
 
 	commit(changed?: Memory[]): MemoryNode[] {
-		if (this.cache) {
+		if(this.cache) {
 			this.cache.rebuild(changed);
 			return this.cache.nodes;
 		}
@@ -187,14 +187,13 @@ class MemoryAccessor {
 		return nodes.filter(n => n.missing).map(n => n.name);
 	}
 
-	/** Cache path uses the KD tree's knn(); raw-array path (no cache available) falls back to a linear cosine scan */
 	search(vector: number[], limit: number): MemoryRef[] {
 		return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit);
 	}
 
 	forget(name: string): boolean {
 		const idx = this.list.findIndex(m => m.name === name);
-		if (idx === -1) return false;
+		if(idx === -1) return false;
 		this.list.splice(idx, 1);
 		this.commit();
 		return true;
@@ -202,7 +201,7 @@ class MemoryAccessor {
 
 	async backfillEmbeddings(llm: any): Promise<number> {
 		const missing = this.list.filter(m => !m.embedding?.length);
-		if (!missing.length) return 0;
+		if(!missing.length) return 0;
 		await Promise.all(missing.map(node => embedMemoryFields(node, llm)));
 		this.commit();
 		return missing.length;
@@ -248,11 +247,11 @@ export class MemoryManager {
 			name: 'memory_recall',
 			description: 'Read the full content of a memory document',
 			args: {
-				name: {type: 'string', description: 'Exact memory name', required: true},
+				name: {type: 'string', description: 'Exact memory name', required: true}
 			},
 			fn: (args: any) => {
-				const mem = this.access(memories).find(args.name);
-				if (!mem) return 'Document not found';
+				const mem = new MemoryAccessor(memories).find(args.name);
+				if(!mem) return 'Document not found';
 				this.touch(mem.name);
 				return mem.content;
 			},
@@ -266,7 +265,7 @@ export class MemoryManager {
 				limit: {type: 'number', description: 'Number of memories to return', default: 1},
 			},
 			fn: async ({query, limit}) => {
-				const mem = await this.recollect(query, memories, limit)
+				const mem = await this.recollect(query, memories, limit);
 				return mem.map(m => `Memory: ${m.name}
 Description: ${m.description}
 Links: ${[...m.links, ...m.backlinks].join(', ')}
@@ -280,17 +279,16 @@ ${m.content}
 	constructor(private llm: any) {}
 
 	static normalize(m?: Memory[] | MemoryCache | MemoryOptions) {
-		if (!m) return null;
+		if(!m) return null;
 		const raw = m instanceof MemoryCache || Array.isArray(m);
 		return raw ? {memory: <Memory[] | MemoryCache>m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m};
 	}
 
-	private access(memories: Memory[] | MemoryCache): MemoryAccessor {
-		return new MemoryAccessor(memories);
-	}
-
 	private stage(node: Memory, block: string): void {
-		this.ensureDoc(node);
+		if(!node.content) {
+			const title = node.name.split('/').pop() ?? node.name;
+			node.content = this.touchHeader(node, `# ${title}\n`);
+		}
 		const body = stripHeader(node.content);
 		const idx = body.indexOf(PENDING_HEADING);
 		const newBody = idx === -1
@@ -299,50 +297,28 @@ ${m.content}
 		node.content = this.touchHeader(node, newBody);
 	}
 
-	private ensureDoc(node: Memory): void {
-		if (node.content) return;
-		const title = node.name.split('/').pop() ?? node.name;
-		node.content = this.touchHeader(node, `# ${title}\n`);
-	}
-
-	private sanitizeDescription(text: string): string {
-		return (text ?? '').replace(/\s+/g, ' ').trim().slice(0, 240);
-	}
-
-	private relink(memories: Memory[], from: string, to: string): void {
-		const pattern = new RegExp(`\\[\\[${escapeRegex(from)}\\]\\]`, 'g');
-		for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`);
-	}
-
-	private normalizeLeaf(name: string): string {
+	private resolveSubject(subject: string, store: MemoryAccessor): string {
+		function normalize(name: string): string {
 			return name.trim().toLowerCase().replace(/\s+/g, ' ');
 		}
 
-	/**
-	 * Resolve a fact-agent proposed subject to an existing node when it's an alias/rename of one.
-	 * Exact match is checked first (cheap, and covers the common case since node names are
-	 * already normalized at creation time). Only falls through to fuzzy alias matching against
-	 * same-root candidates when there's no existing hit — i.e. only on likely-new-doc creation.
-	 */
-	private resolveSubject(subject: string, store: MemoryAccessor): string {
 		const trimmed = subject.trim();
 		const exact = store.find(trimmed);
-		if (exact) return exact.name;
+		if(exact) return exact.name;
 
-		const normalized = this.normalizeLeaf(trimmed);
-		const caseInsensitive = store.list.find(m => this.normalizeLeaf(m.name) === normalized);
-		if (caseInsensitive) return caseInsensitive.name;
+		const normalized = normalize(trimmed);
+		const caseInsensitive = store.list.find(m => normalize(m.name) === normalized);
+		if(caseInsensitive) return caseInsensitive.name;
 
 		const root = trimmed.split('/')[0];
 		const leaf = trimmed.split('/').slice(1).join('/') || trimmed;
 		const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
-		if (!candidates.length) return trimmed;
+		if(!candidates.length) return trimmed;
 
-		// fuzzyMatch requires >=2 terms; pad with an empty string when there's only one candidate
 		const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
 		const probe = leaves.length > 1 ? leaves : [...leaves, ''];
 		const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe);
-		if (max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name;
+		if(max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name;
 
 		return trimmed;
 	}
@@ -353,36 +329,70 @@ ${m.content}
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
-			system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce:
+			system: `Turn this conversation into a persistent memory file by extracting information into organized bullet points
 
-1. Journal recap (single paragraph)
-   - "Captains Log" style record keeping
-   - What was discussed/worked on, decisions, user's events/state/mood, general context
-   - Leave empty only for trivial/empty exchanges/small talk
+Think of this like an Obsidian vault with a clear division of responsibility:
+- The JOURNAL is a timeline. It answers "what happened, and when" and is the only place with a sense of time.
+- ENTITY DOSSIERS are a wiki. They answer "what is currently true about this subject", with no sense of time — only current state.
+- Never blur the two: a one-off event, conversation, or debugging session is a journal entry, not an entity, even if it's detailed.
 
-2. Fact buckets
-   - ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation
-   - NEVER extract greetings, pleasantries, or anything the assistant itself said
-   - Extract the final/end state, not deltas
+1. Journal Log
+- A chronological, skimmable log of what actually happened: real discussions, decisions made, progress on projects, problems worked through
+- This is NOT a transcript, and it is NOT a step-by-step record, its a compressed log of notable events & developments
+- One line per development is usually enough: what was worked on and the outcome, not the blow-by-blow of how
+- Skip small talk and trivial exchanges entirely. Skip anything that's purely a todo item (goes in Todo Tasks) or a durable fact about a subject (goes in Entity Dossiers)
 
-Path assignment rules:
-   - Reuse existing node names whenever possible, including when the subject is an alias/nickname of an existing node (e.g. "Rob" referring to an existing "People/Robert")
-   - Documents should be grouped and named by the root subject
-     - Person → People/Name
-     - Project → Projects/Name
-     - Concept → Concepts/Name
-   - A bug report, its investigation, should be nested and attached to the same root subject node
-   - Tickets/one-off tasks → file under the project/name/component they belong to
-   - Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\`
+2. Todo Tasks
+- Extract concrete tasks the user says need to be done, should be done, or were completed
+- Return the task text and whether it is still todo or is done
+- A completed task should be marked done, not recreated as a new todo
+- Only extract actionable tasks, not general goals or observations
+- Assign each task a subject:
+    - If the task belongs to a persistent entity (a project, a class, etc.), use that entity's exact node name, or a new entity path if it doesn't exist yet
+    - If it's a personal/life task with no entity of its own (reach out to someone, reply to an email, pay a bill, etc.), leave subject as an empty string — it belongs in the journal, not a new document
+
+3. Entity Dossiers
+- Detailed dossiers with all information regarding a subject
+- Record the final/end state, not intermediate changes
+- Ignore assistant claims, guesses, greetings, or temporary details
+- NEVER create a document for something that's only meaningful as a point in time — a single conversation, a one-off decision, a debugging session, a date. That's a journal entry, not an entity
+- identify its HOME ENTITY:
+    - The HOME ENTITY name should always be a [abstract|pro]noun
+    - The grammatical subject/owner of the fact is the strongest clue
+    - Always preference an existing entity over creating a new one
+    - New child entities are appropriate only when they are themselves distinct persistent entities
+    - A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment
+    - Put project facts under the project they belong to, person facts under the person, etc
+
+Example Entity Naming Convention:
+- Projects/[Name]
+- People/[Name]
+- History/[Name]
+- Science/[Name]
+- [Subject]/[Name]
+- Class/[Name]/[Chapter]
+
+Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships
+Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent
 
 Available nodes:
 ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
 ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 			schema: {
 				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
-				buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
+				tasks: {
+					type: 'array', description: 'Concrete tasks mentioned or completed in the conversation.', required: false, items: {
 						type: 'object', items: {
-							subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
+							subject: {type: 'string', description: 'Exact node name / new persistent entity path this task belongs to, or an empty string if this is a personal task with no entity of its own (those go in the journal)', required: true},
+							task: {type: 'string', description: 'Concise actionable task', required: true},
+							done: {type: 'boolean', description: 'Whether the task is completed', required: true},
+						},
+					}
+				},
+				buckets: {
+					type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
+						type: 'object', items: {
+							subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true},
 							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
 						},
 					},
@@ -391,7 +401,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		});
 
 		const buckets = new Map<string, string[]>();
-		for (const bucket of response.buckets ?? []) {
+		for(const bucket of response.buckets ?? []) {
 			const subject = bucket.subject.trim();
 			const facts = buckets.get(subject) ?? [];
 			facts.push(...dedupeFacts(bucket.facts));
@@ -401,10 +411,11 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return {
 			buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
 			journal: (response.journal ?? '').trim(),
+			tasks: response.tasks ?? [],
 		};
 	}
 
-	private getWeekMonday(date: Date = new Date()): string {
+	private getWeekStart(date: Date = new Date()): string {
 		const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
 		const day = d.getUTCDay();
 		const diff = day === 0 ? -6 : 1 - day;
@@ -412,44 +423,92 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 		return d.toISOString().slice(0, 10);
 	}
 
+	private journalDescription(journalName?: string): string {
+		const start = journalName?.split('/').pop() || this.getWeekStart();
+		const d = new Date(`${start}T00:00:00Z`);
+		d.setUTCDate(d.getUTCDate() + 6);
+		const end = d.toISOString().slice(0, 10);
+		return `Log from ${start} - ${end}`;
+	}
+
+	private getIncompleteTodos(content: string): string[] {
+		const body = stripHeader(content);
+		const match = body.match(/## Todo list\n([\s\S]*?)(?=\n## |$)/i);
+		if(!match) return [];
+		return match[1].split('\n')
+			.map(line => line.match(/^\s*-\s*\[([ xX])\]\s+(.+?)\s*$/))
+			.filter((m): m is RegExpMatchArray => !!m && m[1].toLowerCase() !== 'x')
+			.map(m => m[2].trim());
+	}
+
 	private listNodes(memories: Memory[]): MemoryRef[] {
 		return memories.map(m => ({name: m.name, description: m.description}));
 	}
 
-	/** Finds the closest merge candidate via the KD tree's knn() instead of a manual O(n) cosine scan */
-	private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
-		if (!node.embedding?.length || node.name.startsWith('Journal/')) return null;
-		const store = this.access(memories);
+	private async mergeAgent(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory | null> {
+		function factSimilarity(a: Memory, b: Memory): number {
+			if(!a.bodyEmbeddings?.length || !b.bodyEmbeddings?.length) return 0;
+			let best = 0;
+			for(const av of a.bodyEmbeddings) {
+				for(const bv of b.bodyEmbeddings) best = Math.max(best, 1 - cosineDistance(av, bv));
+			}
+			return best;
+		}
 
-		const candidate = store.search(node.embedding, 5)
-			.find(r => r.name !== node.name && !r.name.startsWith('Journal/') && r.distance !== undefined && r.distance <= threshold);
-		if (!candidate) return null;
-		const closest = store.find(candidate.name);
-		if (!closest) return null;
+		if(!node.embedding?.length || node.name.startsWith('Journal/')) return null;
+		const store = new MemoryAccessor(memories);
+		const candidates = store.list
+			.filter(m => m.name !== node.name && !m.name.startsWith('Journal/'))
+			.filter(m => factSimilarity(node, m) >= FACT_SIMILARITY_THRESHOLD);
 
-		const result = await this.mergeAgent(node, closest, options);
-		const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []};
-		merged.content = this.touchHeader(merged, result.content);
-		await embedMemoryFields(merged, this.llm);
+		if(!candidates.length) return null;
+		const closest = candidates.sort((a, b) => factSimilarity(node, b) - factSimilarity(node, a))[0];
+		const result = await this.llm.ask('', {
+			model: options.model,
+			temperature: 0.3,
+			schema: {
+				aContent: {type: 'string', description: 'Updated document A body in markdown, without frontmatter.', required: true},
+				bContent: {type: 'string', description: 'Updated document B body in markdown, without frontmatter.', required: true},
+			},
+			system: `Maintain these two persistent knowledge-base documents like a wiki.
 
-		this.relink(store.list, node.name, merged.name);
-		this.relink(store.list, closest.name, merged.name);
+Do NOT merge, rename, or delete either document. Both represent entities that should remain independently addressable.
 
-		this.queues.get(closest.name)?.request?.abort?.();
-		this.queues.delete(closest.name);
+The documents were selected because their facts may overlap. Your job is to reconcile duplicated information and connect the documents:
+- Decide which document is the HOME for each duplicated fact.
+- Keep the authoritative copy in that home document.
+- In the other document, replace the information with a short preamble and [[WikiLink]] to the home entity explaining the relationship.
+- If the documents are distinct entities but merely related, keep their distinct facts and add useful [[WikiLinks]] between them.
+- Do not delete useful entity-specific facts just because they are similar.
+- Do not invent relationships or facts.
+- Preserve useful history, technical specifics, structure, and existing [[WikiLinks]].
+- Most current truth wins when facts conflict.
+- Keep both documents concise and information-dense.
+- No frontmatter, preamble, filler, or AI commentary.
 
-		store.forget(node.name);
-		store.forget(closest.name);
-		store.list.push(merged);
-		store.commit();
+Document A ("${node.name}"):
+\`\`\`markdown
+${stripHeader(node.content)}
+\`\`\`
 
-		return merged;
+Document B ("${closest.name}"):
+\`\`\`markdown
+${stripHeader(closest.content)}
+\`\`\``,
+		});
+		const a = store.find(node.name);
+		const b = store.find(closest.name);
+		if(!a || !b || !result?.aContent || !result?.bContent) return null;
+		a.content = this.touchHeader(a, result.aContent);
+		b.content = this.touchHeader(b, result.bContent);
+		await Promise.all([embedMemoryFields(a, this.llm), embedMemoryFields(b, this.llm)]);
+		return a;
 	}
 
 	private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
 		const key = node.name;
 		const existing = this.queues.get(key);
-		if (existing) {
+		if(existing) {
 			existing.dirty = true;
 			existing.request?.abort?.();
 			return existing.task;
@@ -457,19 +516,19 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 
 		const entry = {dirty: false, request: null, task: Promise.resolve()};
 		this.queues.set(key, entry);
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		entry.task = (async () => {
-			let current = node, merged = false;
+			let current = node;
 			try {
 				do {
 					entry.dirty = false;
 					await this.docAgent(current, store.list, options, entry);
-					this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
+					this.mergeLock = this.mergeLock.then(() => this.mergeAgent(current, memories, options));
 					const result = await this.mergeLock;
-					if (result) { current = result; merged = true; }
-				} while (entry.dirty);
+					if(result) current = result;
+				} while(entry.dirty);
 			} finally {
-				store.commit(merged ? undefined : [node]);
+				store.commit([node]);
 				this.queues.delete(key);
 			}
 		})();
@@ -479,9 +538,43 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 	private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
 		if(!memories.includes(node)) return;
 		const currentBody = stripHeader(node.content);
+		const journal = node.name.startsWith('Journal/');
+		const system = (journal
+			? `You maintain one persistent journal document
+
+Rewrite the ENTIRE journal, folding "## Pending" into the existing content removing the heading
+
+Journal design:
+- Preserve the chronological daily log
+- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
+- Group information by day under a date heading
+- Keep journal entries high level and concise: what was worked on and the outcome, not a step-by-step record of how — that detail lives in conversation history, not here
+- Use [[WikiLinks]] for persistent entities; don't turn ordinary journal events into entities
+- No frontmatter, preamble, filler, or AI commentary`
+			: `You maintain one persistent knowledge-base entity document
+
+Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished.
+
+Document design:
+- The document represents one persistent entity. Keep information about that entity together and organized into sections
+- Merge any pending information in, newest fact wins conflicts; remove redundant content
+- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections
+- Let the structure fit the entity; there is NO fixed template
+- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
+- Keep the document concise and information-dense without removing useful technical specifics
+- Current truth wins when facts conflict. Preserve older conflict as context, only when it adds useful meaning
+- No frontmatter, preamble, filler, or AI commentary`) + `
+
+Available nodes to link to:
+${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
+
+Current document:
+\`\`\`markdown
+${currentBody}
+\`\`\``;
 		let update;
 		try {
-			for (let i = 0; i < 2 && !update?.content; i++) {
+			for(let i = 0; i < 2 && !update?.content; i++) {
 				const request = this.llm.ask(currentBody, {
 					model: options.model,
 					temperature: 0.3,
@@ -489,104 +582,44 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
-					system: `You are a knowledge base editor maintaining one Obsidian-style document.
-
-If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
-
-Use this loose structure, adapting headings to what the content needs:
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
-
-Rules:
-- Contradictions: "## Pending" holds the newest information — bias toward it. Fold it in as the standing fact and drop the outdated statement, unless the old context adds meaningful nuance (e.g. "previously X, now Y"). This document should read as a source of truth, not an audit log
-- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
-- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
-- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
-- Keep concise, factual, human-readable
-- NO frontmatter, filler, preamble, or AI commentary
-
-Available nodes to link to (don't duplicate their content):
-${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
-
-Current document:
-\`\`\`markdown
-${currentBody}
-\`\`\``,
+					system,
 				});
 				entry.request = request;
 				update = await request;
 			}
-		} catch (err: any) {
-			if (err?.name === 'AbortError') return;
+		} catch(err: any) {
+			if(err?.name === 'AbortError') return;
 			throw err;
 		} finally {
 			entry.request = null;
 		}
 
-		if (!update?.content) return;
-		node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
+		if(!update?.content) return;
+		node.description = node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.name !== 'People/User' ? update.description.replaceAll(/[\n:]/g, '') : 'All information about the current user';
 		node.content = this.touchHeader(node, update.content);
 		await embedMemoryFields(node, this.llm);
 	}
 
-	private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> {
-		const modifiedOf = (m: Memory) => this.parseFrontmatter(m.content).fm.get('modified') || 'unknown';
-
-		return this.llm.ask('', {
-			model: options.model,
-			temperature: 0.3,
-			schema: {
-				name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true},
-				description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
-				content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
-			},
-			system: `You are a knowledge base editor merging two overlapping Obsidian documents into one.
-
-Structure loosely:
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
-
-Combine both documents, resolve duplication. On contradictions, bias toward whichever document was modified more recently; drop the outdated statement unless the old context adds meaningful nuance.
-
-Document A ("${a.name}", last modified ${modifiedOf(a)}):
-\`\`\`markdown
-${stripHeader(a.content)}
-\`\`\`
-
-Document B ("${b.name}", last modified ${modifiedOf(b)}):
-\`\`\`markdown
-${stripHeader(b.content)}
-\`\`\``,
-		});
-	}
-
 	private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
 		const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
-		if (!match) return {fm: new Map(), body: content};
+		if(!match) return {fm: new Map(), body: content};
 		const fm = new Map<string, string>();
-		for (const line of match[1].split('\n')) {
+		for(const line of match[1].split('\n')) {
 			const i = line.indexOf(':');
-			if (i === -1) continue;
+			if(i === -1) continue;
 			const key = line.slice(0, i).trim();
 			const raw = line.slice(i + 1).trim();
 			let value = raw;
-			try { value = JSON.parse(raw); } catch { /* legacy unquoted value, keep raw */ }
+			try { value = JSON.parse(raw); } catch { }
 			fm.set(key, value);
 		}
 		return {fm, body: match[2]};
 	}
 
-	/**
-	 * Writes the code-owned frontmatter block. `body` is passed through stripHeader() first so a
-	 * model that ignores instructions and hallucinates its own `---` block can never corrupt or
-	 * duplicate the real frontmatter — the LLM only ever gets to influence the body.
-	 */
 	private touchHeader(node: Memory, body: string): string {
 		const {fm} = this.parseFrontmatter(node.content);
 		fm.set('name', node.name);
-		fm.set('description', node.description || '');
+		fm.set('description', (node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.description) || 'Persistent memory document');
 		fm.set('modified', new Date().toISOString());
 		return this.writeFrontmatter(fm, stripHeader(body));
 	}
@@ -597,8 +630,8 @@ ${stripHeader(b.content)}
 	}
 
 	decay() {
-		for (const [name, ttl] of this.recentlyTouched) {
-			if (ttl <= 1) this.recentlyTouched.delete(name);
+		for(const [name, ttl] of this.recentlyTouched) {
+			if(ttl <= 1) this.recentlyTouched.delete(name);
 			else this.recentlyTouched.set(name, ttl - 1);
 		}
 	}
@@ -608,11 +641,11 @@ ${stripHeader(b.content)}
 	}
 
 	forget(name: string, memories: Memory[] | MemoryCache): boolean {
-		return this.access(memories).forget(name);
+		return new MemoryAccessor(memories).forget(name);
 	}
 
-	/** Ranks a candidate pool by weighted title/description/body similarity against the query embedding */
-	private rankByFields(query: number[], candidates: Memory[], limit: number): Memory[] {
+	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
+		function rank(query: number[], candidates: Memory[], limit: number): Memory[] {
 			const scored = candidates.map(m => {
 				const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0;
 				const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0;
@@ -624,30 +657,27 @@ ${stripHeader(b.content)}
 			return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory);
 		}
 
-	async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
-		const store = this.access(memories);
-		if (!store.list.length) return [];
-
+		const store = new MemoryAccessor(memories);
+		if(!store.list.length) return [];
 		await store.backfillEmbeddings(this.llm);
 
 		const [e] = await this.llm.embedding(query);
-		if (!e) return [];
+		if(!e) return [];
 
-		// Description embedding is the cheap ANN index key; pull a wider pool then re-rank by field weight
 		const pool = store.search(e.embedding, Math.max(limit * 3, limit));
 		const poolMemories = pool.map(r => store.find(r.name)).filter((m): m is Memory => !!m);
-		const ranked = this.rankByFields(e.embedding, poolMemories, limit);
+		const ranked = rank(e.embedding, poolMemories, limit);
 		const found = new Set<string>(ranked.map(m => m.name));
 
-		if (graphDepth > 0) {
+		if(graphDepth > 0) {
 			let frontier = [...found];
-			for (let depth = 0; depth < graphDepth && frontier.length; depth++) {
+			for(let depth = 0; depth < graphDepth && frontier.length; depth++) {
 				const next: string[] = [];
-				for (const name of frontier) {
+				for(const name of frontier) {
 					const node = store.find(name);
-					if (!node) continue;
-					for (const link of node.links) {
-						if (!found.has(link) && store.find(link)) {
+					if(!node) continue;
+					for(const link of node.links) {
+						if(!found.has(link) && store.find(link)) {
 							found.add(link);
 							next.push(link);
 						}
@@ -666,35 +696,75 @@ ${stripHeader(b.content)}
 		const conversation = history
 			.filter(h => h.role === 'user' || h.role === 'assistant')
 			.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
-		if (!conversation) return [];
+		if(!conversation) return [];
 
 		const uid = `${Date.now()}_${Math.random().toString(36).slice(2)}`;
 		const pending = {role: 'tool', name: 'memory_process', id: uid, content: conversation} as unknown as LLMMessage;
 		history.push(pending);
 
-		const store = this.access(memories);
-		const {buckets, journal} = await this.factAgent(conversation, store, options);
+		const store = new MemoryAccessor(memories);
+		const {buckets, journal, tasks} = await this.factAgent(conversation, store, options);
 		const touched: Memory[] = [];
 
-		if (journal) {
-			const journalName = `Journal/${this.getWeekMonday()}`;
+		const personalTasks = tasks.filter(isPersonalTask);
+		const entityTasks = tasks.filter(t => !isPersonalTask(t));
+
+		if(journal || personalTasks.length) {
+			const journalName = `Journal/${this.getWeekStart()}`;
 			let jnode = store.find(journalName);
-			if (!jnode) {
-				jnode = {name: journalName, description: '', content: '', embedding: [], links: [], backlinks: []};
+			const isNew = !jnode;
+			if(!jnode) {
+				jnode = {
+					name: journalName,
+					description: this.journalDescription(),
+					content: '',
+					embedding: [],
+					links: [],
+					backlinks: [],
+				};
 				store.list.push(jnode);
 			}
-			this.stage(jnode, `### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+
+			const blocks: string[] = [];
+			if(journal) blocks.push(`### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
+			if(isNew) {
+				const previousDate = new Date(`${this.getWeekStart()}T00:00:00Z`);
+				previousDate.setUTCDate(previousDate.getUTCDate() - 7);
+				const previous = store.find(`Journal/${previousDate.toISOString().slice(0, 10)}`);
+				if(previous) {
+					const todos = this.getIncompleteTodos(previous.content);
+					if(todos.length) blocks.push(`${TODO_HEADING}\n${todos.map(task => `- [ ] ${task}`).join('\n')}`);
+				}
+			}
+			if(personalTasks.length) blocks.push(`${TODO_HEADING}\n${personalTasks.map(task => `- [${task.done ? 'x' : ' '}] ${task.task}`).join('\n')}`);
+			if(blocks.length) this.stage(jnode, blocks.join('\n\n'));
 			touched.push(jnode);
 		}
 
-		for (const {subject, facts} of buckets) {
+		const entityStaging = new Map<string, {facts: string[], tasks: MemoryTask[]}>();
+		for(const {subject, facts} of buckets) {
 			const resolved = this.resolveSubject(subject, store);
+			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
+			entry.facts.push(...facts);
+			entityStaging.set(resolved, entry);
+		}
+		for(const task of entityTasks) {
+			const resolved = this.resolveSubject(task.subject, store);
+			const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []};
+			entry.tasks.push(task);
+			entityStaging.set(resolved, entry);
+		}
+
+		for(const [resolved, {facts, tasks: subjectTasks}] of entityStaging) {
 			let node = store.find(resolved);
-			if (!node) {
-				node = {name: resolved, description: '', content: '', embedding: [], links: [], backlinks: []};
+			if(!node) {
+				node = {name: resolved, description: 'Persistent memory document', content: '', embedding: [], links: [], backlinks: []};
 				store.list.push(node);
 			}
-			this.stage(node, facts.map(f => `- ${f}`).join('\n'));
+			const blocks: string[] = [];
+			if(facts.length) blocks.push(facts.map(f => `- ${f}`).join('\n'));
+			if(subjectTasks.length) blocks.push(`${TODO_HEADING}\n${subjectTasks.map(t => `- [${t.done ? 'x' : ' '}] ${t.task}`).join('\n')}`);
+			if(blocks.length) this.stage(node, blocks.join('\n\n'));
 			touched.push(node);
 		}
 
@@ -703,7 +773,7 @@ ${stripHeader(b.content)}
 			this.touch(node.name);
 		}));
 
-		if (touched.length) {
+		if(touched.length) {
 			store.commit(touched);
 			(pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`;
 			Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
@@ -716,7 +786,7 @@ ${stripHeader(b.content)}
 	}
 
 	async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
-		const store = this.access(memories);
+		const store = new MemoryAccessor(memories);
 		const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING));
 		await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
 		store.commit();

src/open-ai.ts

@@ -36,21 +36,49 @@ export class OpenAi extends LLMProvider {
 	private toWire(history: LLMMessage[], system?: string): any[] {
 		const wire: any[] = [];
 		if(system) wire.push({role: 'system', content: system});
-		for(const h of history) {
-			if(h.role === 'tool') {
+
+		for(let i = 0; i < history.length; i++) {
+			const h = history[i];
+
+			if(h.role !== 'tool') {
+				wire.push({role: h.role, content: this.toWireContent(h.content)});
+				continue;
+			}
+
+			const calls: any[] = [];
+			const results: any[] = [];
+
+			while(i < history.length && history[i].role === 'tool') {
+				const tool: any = history[i];
+
+				calls.push({
+					id: tool.id,
+					type: 'function',
+					function: {
+						name: tool.name,
+						arguments: JSON.stringify(tool.args || {})
+					}
+				});
+
+				results.push({
+					role: 'tool',
+					tool_call_id: tool.id,
+					content: tool.error || tool.content || ''
+				});
+
+				i++;
+			}
+
 			wire.push({
 				role: 'assistant',
 				content: null,
-					tool_calls: [{id: h.id, type: 'function', function: {name: h.name, arguments: JSON.stringify(h.args)}}],
-				}, {
-					role: 'tool',
-					tool_call_id: h.id,
-					content: h.error || h.content || '',
+				tool_calls: calls
 			});
-			} else {
-				wire.push({role: h.role, content: this.toWireContent(h.content)});
-			}
+
+			wire.push(...results);
+			i--;
 		}
+
 		return wire;
 	}
 
@@ -60,13 +88,12 @@ export class OpenAi extends LLMProvider {
 			if(!options.history) options.history = [];
 			const history = options.history;
 			if(message) history.push({role: 'user', content: message, timestamp: Date.now()});
-
 			const tools = options.tools || this.ai.options.llm?.tools || [];
 			const requestParams: any = {
 				model: options.model || this.model,
 				stream: !!options.stream,
-				max_completion_tokens: options.maxTokens || this.ai.options.llm?.maxTokens || undefined,
-				temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
+				max_completion_tokens: options.maxTokens ?? this.ai.options.llm?.maxTokens,
+				temperature: options.temperature ?? this.ai.options.llm?.temperature,
 				tools: tools.map(t => ({
 					type: 'function',
 					function: {
@@ -74,8 +101,12 @@ export class OpenAi extends LLMProvider {
 						description: t.description,
 						parameters: {
 							type: 'object',
-							properties: t.args ? objectMap(t.args, (key, value) => ({...value, required: undefined})) : {},
-							required: t.args ? Object.entries(t.args).filter(t => t[1].required).map(t => t[0]) : []
+							properties: t.args
+								? objectMap(t.args, (key, value) => ({...value, required: undefined}))
+								: {},
+							required: t.args
+								? Object.entries(t.args).filter(t => t[1].required).map(t => t[0])
+								: []
 						}
 					}
 				}))
@@ -83,60 +114,106 @@ export class OpenAi extends LLMProvider {
 
 			if(options.schema) {
 				const schema = convertSchema(options.schema);
-				requestParams.response_format = {type: 'json_schema', json_schema: {name: 'response', strict: true, schema}};
+				requestParams.response_format = {
+					type: 'json_schema',
+					json_schema: {name: 'response', strict: true, schema}
+				};
 			}
 			if(options.stream) requestParams.stream_options = {include_usage: true};
 
 			try {
 				let terminal = false;
+				let iteration = 0;
+
 				do {
+					iteration++;
 					requestParams.messages = this.toWire(history.filter(h => h.role !== 'system'), options.system);
 
 					const callStart = Date.now();
-					const resp: any = await this.tokenPool.run(token => this.getClient(token).chat.completions.create(requestParams)).catch(err => {
+					const resp: any = await this.tokenPool.run(token =>
+						this.getClient(token).chat.completions.create(requestParams)
+					).catch(err => {
 						err.message += `\n\nMessages:\n${JSON.stringify(requestParams.messages, null, 2)}`;
 						throw err;
 					});
 
-					let usage: any, msg: any = {content: '', tool_calls: []};
+					let usage: any;
+					let finishReason: string | undefined;
+					let msg: any = {content: '', tool_calls: []};
+					let streamedChars = 0;
+
 					if(options.stream) {
+						let streamCompleted = false;
+						try {
 							for await (const chunk of resp) {
 								if(controller.signal.aborted) break;
 								if(chunk.usage) usage = chunk.usage;
-							if(chunk.choices[0]?.delta?.content) {
-								msg.content += chunk.choices[0].delta.content;
-								options.stream({text: chunk.choices[0].delta.content});
+
+								const choice = chunk.choices?.[0];
+								if(choice?.finish_reason) finishReason = choice.finish_reason;
+
+								if(choice?.delta?.content) {
+									msg.content += choice.delta.content;
+									streamedChars += choice.delta.content.length;
+									options.stream({text: choice.delta.content});
 								}
-							if(chunk.choices[0]?.delta?.tool_calls) {
-								for(const deltaTC of chunk.choices[0].delta.tool_calls) {
-									const existing = msg.tool_calls.find((tc: any) => tc.index === deltaTC.index);
-									if(existing) {
+
+								if(choice?.delta?.tool_calls) {
+									for(const deltaTC of choice.delta.tool_calls) {
+										const index = deltaTC.index ?? msg.tool_calls.length;
+										let existing = msg.tool_calls.find((tc: any) => tc.index === index);
+
+										if(!existing) {
+											existing = {index, id: '', function: {name: '', arguments: ''}};
+											msg.tool_calls.push(existing);
+										}
+
 										if(deltaTC.id) existing.id = deltaTC.id;
 										if(deltaTC.function?.name) existing.function.name = deltaTC.function.name;
 										if(deltaTC.function?.arguments) existing.function.arguments += deltaTC.function.arguments;
-									} else {
-										msg.tool_calls.push({
-											index: deltaTC.index,
-											id: deltaTC.id || '',
-											function: {name: deltaTC.function?.name || '', arguments: deltaTC.function?.arguments || ''}
-										});
 									}
 								}
 							}
+
+							streamCompleted = true;
+						} catch(err) {
+							if(!controller.signal.aborted) throw err;
 						}
+
+						if(streamCompleted && !finishReason) finishReason = msg.tool_calls.length ? 'tool_calls' : 'stop';
 					} else {
 						usage = resp.usage;
+						finishReason = resp.choices[0].finish_reason;
 						msg = resp.choices[0].message;
 					}
+
 					const duration = Date.now() - callStart;
 					const tps = usage?.completion_tokens && duration > 0 ? usage.completion_tokens / (duration / 1000) : 0;
 
+					if(finishReason === 'length' && !controller.signal.aborted) {
+						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
+						throw new Error(`[OpenAI] Response hit token limit before completing`);
+					}
+
+					if(!finishReason && !controller.signal.aborted) {
+						throw new Error('[OpenAI] Completion ended without a usable response');
+					}
+
 					const toolCalls = msg.tool_calls || [];
+
 					if(toolCalls.length && !controller.signal.aborted) {
 						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
 
 						const entries = toolCalls.map((tc: any) => {
-							const entry: any = {role: 'tool', id: tc.id, name: tc.function.name, args: JSONAttemptParse(tc.function.arguments, {}), content: undefined, timestamp: Date.now()};
+							const entry: any = {
+								role: 'tool',
+								id: tc.id,
+								name: tc.function.name,
+								args: JSONAttemptParse(tc.function.arguments, {}),
+								content: undefined,
+								timestamp: Date.now()
+							};
+
 							history.push(entry);
 							return {tc, entry};
 						});
@@ -144,12 +221,13 @@ export class OpenAi extends LLMProvider {
 						await Promise.all(entries.map(async ({tc, entry}: any) => {
 							const tool = tools.find(findByProp('name', tc.function.name));
 							if(options.stream) options.stream({tool: tc.function.name});
-							if(!tool) { entry.error = 'Tool not found'; return; }
+							if(!tool) return entry.error = 'Tool not found';
 							try {
 								const toolStream = options.stream && ((chunk: any) => {
-									if(chunk.done) { terminal = true; return; }
+									if(chunk.done) return;
 									options.stream!(chunk);
 								});
+
 								const result = await tool.fn(entry.args, toolStream, this.ai, tc.id);
 								entry.content = typeof result === 'object' ? JSONSanitize(result) : result;
 							} catch(err: any) {
@@ -164,7 +242,6 @@ export class OpenAi extends LLMProvider {
 				} while(!terminal && !controller.signal.aborted);
 
 				if(options.stream) options.stream({done: true});
-
 				const turnStart = history.map(h => h.role).lastIndexOf('user');
 				const finalContent = history.slice(turnStart + 1).reduce((str, h) => h.role === 'assistant' ? str + (h.content || '') : str, '').trim();
 				res(options.schema ? JSONAttemptParse(finalContent, finalContent) : finalContent);