Entity based notes

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.6.7",
+	"version": "1.6.11",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/memory.ts

@@ -8,13 +8,6 @@ const MERGE_THRESHOLD = 0.12;
 const PENDING_HEADING = '## Pending';
 const TREE_TOMBSTONE_LIMIT = 0.25;
 const ALIAS_MATCH_THRESHOLD = 0.55;
-const GENERIC_TEMPLATE = `# {{Title}}
-
-## Summary
-
-## Details
-
-## Related`;
 
 export type Memory = {
 	name: string;
@@ -338,7 +331,6 @@ ${m.content}
 		const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
 		if (!candidates.length) return trimmed;
 
-		// fuzzyMatch requires >=2 terms; pad with an empty string when there's only one candidate
 		const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
 		const probe = leaves.length > 1 ? leaves : [...leaves, ''];
 		const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe);
@@ -353,27 +345,35 @@ ${m.content}
 		const response = await this.llm.ask(conversation, {
 			model: options.model,
 			temperature: 0.2,
-			system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce:
+			system: `Extract durable memory from this conversation
 
-1. Journal recap (single paragraph)
+1. Journal recap
-   - "Captains Log" style record keeping
+- Brief "Captain's Log" of what happened, including useful context, decisions, or events
-   - What was discussed/worked on, decisions, user's events/state/mood, general context
+- Leave empty for trivial exchanges
-   - Leave empty only for trivial/empty exchanges/small talk
 
 2. Fact buckets
-   - ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation
+- Extract only durable facts explicitly stated by the USER
-   - NEVER extract greetings, pleasantries, or anything the assistant itself said
+- Record the final/end state, not intermediate changes
-   - Extract the final/end state, not deltas
+- Do not extract assistant claims, guesses, greetings, or temporary conversation details
 
-Path assignment rules:
+For each fact, identify its HOME ENTITY:
-   - Reuse existing node names whenever possible, including when the subject is an alias/nickname of an existing node (e.g. "Rob" referring to an existing "People/Robert")
+- The HOME ENTITY name should always be a [abstract|pro]noun
-   - Documents should be grouped and named by the root subject
+- The grammatical subject/owner of the fact is the strongest clue
-     - Person → People/Name
+- Prefer an existing entity over creating a new one
-     - Project → Projects/Name
+- A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment
-     - Concept → Concepts/Name
+- Put project facts under the project they belong to, person facts under the person, etc
-   - A bug report, its investigation, should be nested and attached to the same root subject node
+- New child entities are appropriate only when they are themselves distinct persistent entities
-   - Tickets/one-off tasks → file under the project/name/component they belong to
+
-   - Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\`
+Example Paths:
+- Projects/[Name]
+- People/[Name]
+- History/[Name]
+- Science/[Name]
+- [Subject]/[Name]
+- Class/[Name]/[Child]
+
+Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships
+Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent
 
 Available nodes:
 ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
@@ -382,7 +382,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 				journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
 				buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
 						type: 'object', items: {
-							subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
+							subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true},
 							facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
 						},
 					},
@@ -489,24 +489,22 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
 						description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
 						content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
 					},
-					system: `You are a knowledge base editor maintaining one Obsidian-style document.
+					system: `You maintain one persistent knowledge-base document
 
-If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
+Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished
 
-Use this loose structure, adapting headings to what the content needs:
+Document design:
-\`\`\`markdown
+- The document represents one entity. Keep information about that entity together
-${GENERIC_TEMPLATE}
+- Let the structure fit the entity; there is NO fixed template
-\`\`\`
+- Preserve useful existing headings and organization. Don't redesign the document without reason
+- Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts
+- Keep the document concise and information-dense without removing useful technical specifics
+- Current truth wins when facts conflict. Preserve older context only when it adds useful meaning
+- Use [[WikiLinks]] for specific related entities; don't create redundant content for linked entities
+- Avoid generic filler sections such as Notes, Miscellaneous, Recent, Updates, or Conversation
+- No frontmatter, preamble, filler, or AI commentary
 
-Rules:
+Available nodes to link to:
-- Contradictions: "## Pending" holds the newest information — bias toward it. Fold it in as the standing fact and drop the outdated statement, unless the old context adds meaningful nuance (e.g. "previously X, now Y"). This document should read as a source of truth, not an audit log
-- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
-- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
-- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
-- Keep concise, factual, human-readable
-- NO frontmatter, filler, preamble, or AI commentary
-
-Available nodes to link to (don't duplicate their content):
 ${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
 
 Current document:
@@ -537,18 +535,22 @@ ${currentBody}
 			model: options.model,
 			temperature: 0.3,
 			schema: {
-				name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true},
+				name: {type: 'string', description: 'Canonical path for the merged entity', required: true},
 				description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
 				content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
 			},
-			system: `You are a knowledge base editor merging two overlapping Obsidian documents into one.
+			system: `Determine whether these two documents represent the SAME persistent entity.
 
-Structure loosely:
+Similarity of subject matter is NOT enough. Do not merge documents merely because they discuss the same project, person, technology, topic, or related work.
-\`\`\`markdown
-${GENERIC_TEMPLATE}
-\`\`\`
 
-Combine both documents, resolve duplication. On contradictions, bias toward whichever document was modified more recently; drop the outdated statement unless the old context adds meaningful nuance.
+Merge only when the evidence indicates they are duplicate identities, aliases, renamed entities, or two documents accidentally created for the same real-world entity. If they are distinct entities, they must remain separate.
+
+If they are the same entity:
+- Choose the canonical/most established path.
+- Combine their information into one document and remove duplication.
+- Preserve useful structure, technical specifics, history, and [[WikiLinks]].
+- Prefer newer information when facts conflict.
+- Return the canonical entity name and the fully reconciled document.
 
 Document A ("${a.name}", last modified ${modifiedOf(a)}):
 \`\`\`markdown

src/open-ai.ts

@@ -98,18 +98,21 @@ export class OpenAi extends LLMProvider {
 						throw err;
 					});
 
-					let usage: any, msg: any = {content: '', tool_calls: []};
+					let usage: any, finishReason: string | undefined, msg: any = {content: '', tool_calls: []};
 					if(options.stream) {
 						for await (const chunk of resp) {
 							if(controller.signal.aborted) break;
 							if(chunk.usage) usage = chunk.usage;
+							if(chunk.choices[0]?.finish_reason) finishReason = chunk.choices[0].finish_reason;
 							if(chunk.choices[0]?.delta?.content) {
 								msg.content += chunk.choices[0].delta.content;
 								options.stream({text: chunk.choices[0].delta.content});
 							}
 							if(chunk.choices[0]?.delta?.tool_calls) {
 								for(const deltaTC of chunk.choices[0].delta.tool_calls) {
-									const existing = msg.tool_calls.find((tc: any) => tc.index === deltaTC.index);
+									const existing = deltaTC.index != null
+										? msg.tool_calls.find((tc: any) => tc.index === deltaTC.index)
+										: (deltaTC.id ? msg.tool_calls.find((tc: any) => tc.id === deltaTC.id) : undefined);
 									if(existing) {
 										if(deltaTC.id) existing.id = deltaTC.id;
 										if(deltaTC.function?.name) existing.function.name = deltaTC.function.name;
@@ -126,11 +129,21 @@ export class OpenAi extends LLMProvider {
 						}
 					} else {
 						usage = resp.usage;
+						finishReason = resp.choices[0].finish_reason;
 						msg = resp.choices[0].message;
 					}
 					const duration = Date.now() - callStart;
 					const tps = usage?.completion_tokens && duration > 0 ? usage.completion_tokens / (duration / 1000) : 0;
 
+					if(finishReason === 'length' && !controller.signal.aborted) {
+						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
+						throw new Error(`[OpenAI] Response hit token limit before completing`);
+					}
+
+					if(!finishReason && !controller.signal.aborted) {
+						throw new Error('[OpenAI] Stream ended prematurely - connection likely dropped');
+					}
+
 					const toolCalls = msg.tool_calls || [];
 					if(toolCalls.length && !controller.signal.aborted) {
 						if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps});
@@ -147,7 +160,7 @@ export class OpenAi extends LLMProvider {
 							if(!tool) { entry.error = 'Tool not found'; return; }
 							try {
 								const toolStream = options.stream && ((chunk: any) => {
-									if(chunk.done) { terminal = true; return; }
+									if(chunk.done) return;
 									options.stream!(chunk);
 								});
 								const result = await tool.fn(entry.args, toolStream, this.ai, tc.id);