Entity based notes

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.8",
+	"version": "1.6.11",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/memory.ts

@@ -8,13 +8,6 @@ const MERGE_THRESHOLD = 0.12;
 const PENDING_HEADING = '## Pending';
 const TREE_TOMBSTONE_LIMIT = 0.25;
 const ALIAS_MATCH_THRESHOLD = 0.55;
-const GENERIC_TEMPLATE = `# {{Title}}
-
-## Summary
-
-## Details
-
-## Related`;
 
 export type Memory = {
 	name: string;
@@ -338,7 +331,6 @@ ${m.content}
 		const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
 		if (!candidates.length) return trimmed;
 
-		// fuzzyMatch requires >=2 terms; pad with an empty string when there's only one candidate
 		const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
 		const probe = leaves.length > 1 ? leaves : [...leaves, ''];
 		const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe);
@@ -353,35 +345,35 @@ ${m.content}
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
-			system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce:
+			system: `Extract durable memory from this conversation
 
-1. Journal recap (single paragraph)
-   - "Captains Log" style record keeping
-   - What was discussed/worked on, decisions, user's events/state/mood, general context
-   - Leave empty only for trivial/empty exchanges/small talk
+1. Journal recap
+- Brief "Captain's Log" of what happened, including useful context, decisions, or events
+- Leave empty for trivial exchanges
 
 2. Fact buckets
-   - ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation
-   - NEVER extract greetings, pleasantries, or anything the assistant itself said
-   - Extract the final/end state, not deltas
+- Extract only durable facts explicitly stated by the USER
+- Record the final/end state, not intermediate changes
+- Do not extract assistant claims, guesses, greetings, or temporary conversation details
 
-Path assignment (entity) rules:
-   - Use the owning entity of the fact (even if implied): "New bug on project 51 -> Projects/51"
-   - When multiple facts relate to the same entity, pick a primary owner and wikilink related entities
-   - Reuse existing entities when the owner already has a node
-   - Always group under consistent entity roots (always plural):
-     - Projects/[Name] for all initiatives
-     - People/[Name] for all individuals
-     - History/[Name] for all historical figures/events
-     - Science/[Name] for all scientific concepts
-   - Child entities nest under their parent entity:
-     - Projects/51/Memory System, Projects/51/Bug-XYZ, not Bugs/51
-     - Science/AI/Model-X, not Model-X/AI
+For each fact, identify its HOME ENTITY:
+- The HOME ENTITY name should always be a [abstract|pro]noun
+- The grammatical subject/owner of the fact is the strongest clue
+- Prefer an existing entity over creating a new one
+- A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment
+- Put project facts under the project they belong to, person facts under the person, etc
+- New child entities are appropriate only when they are themselves distinct persistent entities
 
-Wikilink rules:
-   - Use [[WikiLinks]] to connect related entities (e.g., [[Projects/51]], [[People/Robert]])
-   - Only link specific, existing or implied entity paths — skip generic terms
-   - Don't over-link: each link should add clarity or context, not noise
+Example Paths:
+- Projects/[Name]
+- People/[Name]
+- History/[Name]
+- Science/[Name]
+- [Subject]/[Name]
+- Class/[Name]/[Child]
+
+Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships
+Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent
 
 Available nodes:
 ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
@@ -390,7 +382,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
 				buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
 						type: 'object', items: {
-							subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
+							subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true},
 							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
 						},
 					},
@@ -497,24 +489,22 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
-					system: `You are a knowledge base editor maintaining one Obsidian-style document.
+					system: `You maintain one persistent knowledge-base document
 
-If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
+Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished
 
-Use this loose structure, adapting headings to what the content needs:
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
+Document design:
+- The document represents one entity. Keep information about that entity together
+- Let the structure fit the entity; there is NO fixed template
+- Preserve useful existing headings and organization. Don't redesign the document without reason
+- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
+- Keep the document concise and information-dense without removing useful technical specifics
+- Current truth wins when facts conflict. Preserve older context only when it adds useful meaning
+- Use [[WikiLinks]] for specific related entities; don't create redundant content for linked entities
+- Avoid generic filler sections such as Notes, Miscellaneous, Recent, Updates, or Conversation
+- No frontmatter, preamble, filler, or AI commentary
 
-Rules:
-- Contradictions: "## Pending" holds the newest information — bias toward it. Fold it in as the standing fact and drop the outdated statement, unless the old context adds meaningful nuance (e.g. "previously X, now Y"). This document should read as a source of truth, not an audit log
-- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
-- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
-- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
-- Keep concise, factual, human-readable
-- NO frontmatter, filler, preamble, or AI commentary
-
-Available nodes to link to (don't duplicate their content):
+Available nodes to link to:
 ${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
 
 Current document:
@@ -545,18 +535,22 @@ ${currentBody}
 			model: options.model,
 			temperature: 0.3,
 			schema: {
-				name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true},
+				name: {type: 'string', description: 'Canonical path for the merged entity', required: true},
 				description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
 				content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
 			},
-			system: `You are a knowledge base editor merging two overlapping Obsidian documents into one.
+			system: `Determine whether these two documents represent the SAME persistent entity.
 
-Structure loosely:
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
+Similarity of subject matter is NOT enough. Do not merge documents merely because they discuss the same project, person, technology, topic, or related work.
 
-Combine both documents, resolve duplication. On contradictions, bias toward whichever document was modified more recently; drop the outdated statement unless the old context adds meaningful nuance.
+Merge only when the evidence indicates they are duplicate identities, aliases, renamed entities, or two documents accidentally created for the same real-world entity. If they are distinct entities, they must remain separate.
+
+If they are the same entity:
+- Choose the canonical/most established path.
+- Combine their information into one document and remove duplication.
+- Preserve useful structure, technical specifics, history, and [[WikiLinks]].
+- Prefer newer information when facts conflict.
+- Return the canonical entity name and the fully reconciled document.
 
 Document A ("${a.name}", last modified ${modifiedOf(a)}):
 \`\`\`markdown

src/open-ai.ts

@@ -41,7 +41,7 @@ export class OpenAi extends LLMProvider {
 				wire.push({
 					role: 'assistant',
 					content: null,
-					tool_calls: [{id: h.id, type: 'function', function: {name: h.name, arguments: JSON.stringify(h.args)}],
+					tool_calls: [{id: h.id, type: 'function', function: {name: h.name, arguments: JSON.stringify(h.args)}}],
 				}, {
 					role: 'tool',
 					tool_call_id: h.id,
@@ -98,144 +98,85 @@ export class OpenAi extends LLMProvider {
 						throw err;
 					});
 
-					let usage: any, msg: any = {content: '', tool_calls: []};
+					let usage: any, finishReason: string | undefined, msg: any = {content: '', tool_calls: []};
 					if(options.stream) {
-						const toolCallState: Record<number, any> = {};
-						let pendingToolCalls = 0;
-
 						for await (const chunk of resp) {
 							if(controller.signal.aborted) break;
-
 							if(chunk.usage) usage = chunk.usage;
-
-							// Handle content deltas
+							if(chunk.choices[0]?.finish_reason) finishReason = chunk.choices[0].finish_reason;
 							if(chunk.choices[0]?.delta?.content) {
 								msg.content += chunk.choices[0].delta.content;
-								options.stream?.({text: chunk.choices[0].delta.content});
+								options.stream({text: chunk.choices[0].delta.content});
 							}
-
-							// Handle tool_call deltas
 							if(chunk.choices[0]?.delta?.tool_calls) {
 								for(const deltaTC of chunk.choices[0].delta.tool_calls) {
-									const tc = toolCallState[deltaTC.index];
-
-									if(!tc && deltaTC.id) {
-										// New tool call delta
-										toolCallState[deltaTC.index] = {
-											id: deltaTC.id,
-											name: deltaTC.function?.name || '',
-											arguments: deltaTC.function?.arguments || '',
-											complete: false
-										};
+									const existing = deltaTC.index != null
+										? msg.tool_calls.find((tc: any) => tc.index === deltaTC.index)
+										: (deltaTC.id ? msg.tool_calls.find((tc: any) => tc.id === deltaTC.id) : undefined);
+									if(existing) {
+										if(deltaTC.id) existing.id = deltaTC.id;
+										if(deltaTC.function?.name) existing.function.name = deltaTC.function.name;
+										if(deltaTC.function?.arguments) existing.function.arguments += deltaTC.function.arguments;
+									} else {
 										msg.tool_calls.push({
 											index: deltaTC.index,
 											id: deltaTC.id || '',
 											function: {name: deltaTC.function?.name || '', arguments: deltaTC.function?.arguments || ''}
 										});
-										pendingToolCalls++;
-									} else if(tc && deltaTC.function?.name) {
-										// Update existing tool call
-										if(deltaTC.id) tc.id = deltaTC.id;
-										if(deltaTC.function.name) tc.name = deltaTC.function.name;
-										if(deltaTC.function.arguments) tc.arguments += deltaTC.function.arguments;
-									}
-
-									if(tc && deltaTC.function?.arguments && !tc.complete) {
-										tc.complete = true;
-										pendingToolCalls--;
 									}
 								}
 							}
-
-							// Execute completed tools immediately when all deltas are received
-							if(!pendingToolCalls && msg.tool_calls.length > 0) {
-								const completedTools: any[] = [];
-
-								for(const tc of msg.tool_calls) {
-									const tcState = toolCallState[tc.index];
-									if(!tcState) continue;
-
-									const entry: any = {role: 'tool', id: tc.id, name: tcState.name, args: JSONAttemptParse(tcState.arguments, {}), content: undefined, timestamp: Date.now()};
-									history.push(entry);
-									completedTools.push({tc, entry});
-
-									const tool = tools.find(findByProp('name', tcState.name));
-									if(options.stream) options.stream?.({tool: tcState.name});
-
-									if(!tool) {
-										entry.error = 'Tool not found';
-										continue;
-									}
-
-									try {
-										const toolStream = options.stream && ((chunk: any) => {
-											if(chunk.done) { terminal = true; return; }
-											options.stream?.(chunk);
-										});
-										const result = await tool.fn(entry.args, toolStream, this.ai, tc.id);
-										entry.content = typeof result === 'object' ? JSONSanitize(result) : result;
-									} catch(err: any) {
-										entry.error = err?.message || err?.toString() || 'Unknown';
-									}
-								}
-
-								if(options.stream) options.stream?.({done: true});
-
-								for(const {entry} of completedTools) {
-									if(entry.error) {
-										options.stream?.({error: entry.error});
-									} else if(entry.content) {
-										options.stream?.({toolResult: true});
-									}
-								}
-
-								msg.tool_calls = []; // Clear after execution
-							}
-						}
-
-						// Handle finish_reason for proper termination detection
-						if(chunk.choices[0]?.finish_reason) {
-							const finishReason = chunk.choices[0].finish_reason;
-							if(finishReason === 'stop' || finishReason === 'tool_calls' || finishReason === 'length') {
-								terminal = true;
-							}
 						}
 					} else {
 						usage = resp.usage;
+						finishReason = resp.choices[0].finish_reason;
 						msg = resp.choices[0].message;
-						if(msg.tool_calls) {
-							// Non-streaming: execute tools immediately
-							for(const tc of msg.tool_calls) {
-								const entry: any = {role: 'tool', id: tc.id, name: tc.function.name, args: JSONAttemptParse(tc.function.arguments, {}), content: undefined, timestamp: Date.now()};
-								history.push(entry);
-
-								const tool = tools.find(findByProp('name', tc.function.name));
-								if(!tool) {
-									entry.error = 'Tool not found';
-									continue;
-								}
-
-								try {
-									const result = await tool.fn(entry.args, undefined, this.ai, tc.id);
-									entry.content = typeof result === 'object' ? JSONSanitize(result) : result;
-								} catch(err: any) {
-									entry.error = err?.message || err?.toString() || 'Unknown';
-								}
-							}
-						}
 					}
-
 					const duration = Date.now() - callStart;
 					const tps = usage?.completion_tokens && duration > 0 ? usage.completion_tokens / (duration / 1000) : 0;
 
-					// Capture assistant messages (before or after tools)
-					if(msg.content?.trim()) {
-						history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
+					if(finishReason === 'length' && !controller.signal.aborted) {
+						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
+						throw new Error(`[OpenAI] Response hit token limit before completing`);
 					}
 
+					if(!finishReason && !controller.signal.aborted) {
+						throw new Error('[OpenAI] Stream ended prematurely - connection likely dropped');
+					}
+
+					const toolCalls = msg.tool_calls || [];
+					if(toolCalls.length && !controller.signal.aborted) {
+						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
+
+						const entries = toolCalls.map((tc: any) => {
+							const entry: any = {role: 'tool', id: tc.id, name: tc.function.name, args: JSONAttemptParse(tc.function.arguments, {}), content: undefined, timestamp: Date.now()};
+							history.push(entry);
+							return {tc, entry};
+						});
+
+						await Promise.all(entries.map(async ({tc, entry}: any) => {
+							const tool = tools.find(findByProp('name', tc.function.name));
+							if(options.stream) options.stream({tool: tc.function.name});
+							if(!tool) { entry.error = 'Tool not found'; return; }
+							try {
+								const toolStream = options.stream && ((chunk: any) => {
+									if(chunk.done) return;
+									options.stream!(chunk);
+								});
+								const result = await tool.fn(entry.args, toolStream, this.ai, tc.id);
+								entry.content = typeof result === 'object' ? JSONSanitize(result) : result;
+							} catch(err: any) {
+								entry.error = err?.message || err?.toString() || 'Unknown';
+							}
+						}));
+					} else {
+						terminal = true;
+						const text = (msg.content || '').trim();
+						if(text) history.push({role: 'assistant', content: text, timestamp: Date.now(), duration, tps});
+					}
 				} while(!terminal && !controller.signal.aborted);
 
-				if(options.stream) options.stream?.({done: true});
+				if(options.stream) options.stream({done: true});
 
 				const turnStart = history.map(h => h.role).lastIndexOf('user');
 				const finalContent = history.slice(turnStart + 1).reduce((str, h) => h.role === 'assistant' ? str + (h.content || '') : str, '').trim();