diff --git a/package.json b/package.json index 60d18d1..c50e101 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/ai-utils", - "version": "1.6.3", + "version": "1.6.4", "description": "AI Utility library", "author": "Zak Timson", "license": "MIT", diff --git a/src/memory.ts b/src/memory.ts index b7e2366..3083e47 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -107,8 +107,8 @@ export class MemoryCache { } update(memory: Memory): void { - const idx = this.memories.findIndex(m => m.name === memory.name); - if (idx !== -1) this.memories[idx] = memory; + const existing = this.memories.find(m => m.name === memory.name); + if (existing) Object.assign(existing, memory); else this.memories.push(memory); this.rebuild(); } @@ -169,7 +169,7 @@ class MemoryAccessor { const missing = this.list.filter(m => !m.embedding?.length); if (!missing.length) return 0; await Promise.all(missing.map(async node => { - const [e] = await llm.embedding(node.content); + const [e] = await llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim()); if (e) node.embedding = e.embedding; })); this.commit(); @@ -288,42 +288,37 @@ ${m.content} const response = await this.llm.ask(conversation, { model: options.model, temperature: 0.2, - system: `You are a fact extractor to build obsidian knowledge vaults. -Analyze this conversation and produce two things: a journal recap and any long-term facts worth filing. + system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce: -Journal recap rules: -- Write a short day-to-day recap of this conversation: what was discussed/worked on, decisions still in flux, the user's state/mood if mentioned, general context -- Write something whenever there was any meaningful activity, even if nothing below qualifies as a hard fact -- Leave it empty only for trivial/empty exchanges (greetings, pleasantries, nothing happened) +1. Journal recap (single paragraph) + - "Captains Log" style record keeping + - What was discussed/worked on, decisions, user's events/state/mood, general context + - Leave empty only for trivial/empty exchanges/small talk -Fact bucket rules: -- Always extract facts that the user explicitly told you to remember -- ONLY extract current facts the USER explicitly stated about themselves, their work, projects or decisions that were MADE during this conversation -- DO NOT extract greetings, pleasantries, or generic exchanges -- DO NOT extract deltas or changes in facts; ONLY the end fact -- DO NOT extract anything the AI/assistant itself said -- If nothing worth remembering was said, return an empty buckets array +2. Fact buckets + - ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation + - NEVER extract greetings, pleasantries, or anything the assistant itself said + - Extract the final/end state, not deltas -When extracting facts, you MUST also decide the exact destination path: -- Reuse node names (including ghost) as much as possible IF the fact belongs there -- All information primarily about the user should go under "People/User" -- NEVER use a generic label like "Technical Issue", "Technical Decision", "Technical Requirement", "Bug", "Feature Request" as a subject — these fragment the vault. Every fact belongs to a concrete subject: a person, project, or system. -- A bug report, its investigation, and its eventual fix are the SAME topic — reuse the one node for the subject, don't fork a new one per event -- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide), named after the subject, not the event — you are not limited to any fixed list of collections +Path assignment rules: + - Reuse existing node names whenever possible + - Documents should be grouped and named by the root subject + - Person → People/Name + - Project → Projects/Name + - Concept → Concepts/Name + - A bug report, its investigation, should be nested and attached to the same root subject node + - Tickets/one-off tasks → file under the project/name/component they belong to + - Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\` Available nodes: ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'} ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, schema: { - journal: {type: 'string', description: 'Short day-to-day recap of this conversation, see rules above. Empty string if nothing happened.', required: false}, - buckets: {type: 'array', description: 'Groups of facts to remember, each assigned to a different node. Return an empty array if there is nothing worth storing in an obsidian vault', items: { + journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false}, + buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: { type: 'object', items: { - subject: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true}, - facts: { - type: 'array', - description: 'Facts to store at this destination', - items: {type: 'string', description: 'A single fact'}, - }, + subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true}, + facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}}, }, }, }, @@ -360,7 +355,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, * the other. Journals are exempt — they're partitioned by date, not topic, and merging across weeks * would wreck the timeline. Returns 'merged' if `node` absorbed another (caller should re-run the doc * agent), 'absorbed' if `node` itself got folded away (caller should stop touching it), or null. */ - private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<'merged' | 'absorbed' | null> { + private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise { if (!node.embedding?.length || node.name.startsWith('Journal/')) return null; const store = this.access(memories); @@ -372,16 +367,24 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, } if (!closest || closestDist > threshold) return null; - const weight = (m: Memory) => m.content.length + (m.links.length + m.backlinks.length) * 200; - const [primary, secondary] = weight(node) >= weight(closest) ? [node, closest] : [closest, node]; + const result = await this.mergeAgent(node, closest, options); + const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []}; + merged.content = this.touchHeader(merged, result.content); + const [e] = await this.llm.embedding(`${merged.description}\n\n${result.content}`.trim()); + if (e) merged.embedding = e.embedding; - this.stage(primary, `### Merged from [[${secondary.name}]]\n\n${stripHeader(secondary.content)}`); - this.relink(store.list, secondary.name, primary.name); - store.forget(secondary.name); + this.relink(store.list, node.name, merged.name); + this.relink(store.list, closest.name, merged.name); - if (primary === node) return 'merged'; - this.reconcile(primary, memories, options).catch(() => {}); - return 'absorbed'; + this.queues.get(closest.name)?.request?.abort?.(); + this.queues.delete(closest.name); + + store.forget(node.name); + store.forget(closest.name); + store.list.push(merged); + store.commit(); + + return merged; } private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise { @@ -397,12 +400,12 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, this.queues.set(key, entry); const store = this.access(memories); entry.task = (async () => { + let current = node; do { entry.dirty = false; - await this.docAgent(node, store.list, options, entry); - const merge = await this.checkMerge(node, memories, options); - if (merge === 'absorbed') break; - if (merge === 'merged') entry.dirty = true; + await this.docAgent(current, store.list, options, entry); + const merged = await this.checkMerge(current, memories, options); + if (merged) { current = merged; entry.dirty = true; } } while (entry.dirty); })().finally(() => { this.queues.delete(key); @@ -412,8 +415,8 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, } private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise { + if(!memories.includes(node)) return; const currentBody = stripHeader(node.content); - const isJournal = node.name.startsWith('Journal/'); let update; try { for (let i = 0; i < 2 && !update?.content; i++) { @@ -421,30 +424,29 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, model: options.model, temperature: 0.3, schema: { - description: {type: 'string', description: 'One factual sentence describing what this document contains. Never a thought process, reasoning, or meta-commentary about the task.', required: true}, + description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true}, content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true}, }, - system: `You are a knowledge base editor maintaining one document in an Obsidian-style vault. + system: `You are a knowledge base editor maintaining one Obsidian-style document. -If the document has a "${PENDING_HEADING}" section, it holds new material to integrate — bullet facts, a journal entry, or an entire document merged in from elsewhere (marked "### Merged from [[X]]" or "### {date}"). Fold all of it into the appropriate part of the document, resolving overlap and duplication, then remove the "${PENDING_HEADING}" section entirely. If there is no such section, just tidy the document per the rules below. +If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below. -Structure: follow this generic shape loosely, adapting section names/order to what the content actually needs: -\`\`\`markdown -${GENERIC_TEMPLATE} -\`\`\` +Use this loose structure, adapting headings to what the content needs: -${isJournal - ? `This document is a journal — a chronological day-to-day log. Keep entries ordered as a timeline under "### {date}" headings. NEVER delete or "resolve away" older entries in favor of newer ones — history stays, only clean up grammar/clarity within an entry.` - : `Resolve contradictions: newer facts always win — delete the outdated statement entirely, never keep both.`} +# Title +## Summary +## Details +## Related -Formatting rules: -- Use Obsidian-style markdown: # headings, **bold** for emphasis, bullet & numbered lists for grouped 1D data, tables for 2D data -- Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]] -- Create links for specific entities (person, place, project, program) and abstract concepts, but skip generics (car, red, dog) -- Keep the document concise, factual, and human-readable -- Do not add frontmatter blocks, filler, preamble, or AI commentary +Rules: +- Contradictions: newer facts always win — delete outdated statements entirely +- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history +- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data +- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics +- Keep concise, factual, human-readable +- NO frontmatter, filler, preamble, or AI commentary -Other nodes in the vault (link to these instead of duplicating their content): +Available nodes to link to (don't duplicate their content): ${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'} Current document: @@ -465,10 +467,42 @@ ${currentBody} if (!update?.content) return; node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user'; node.content = this.touchHeader(node, update.content); - const [e] = await this.llm.embedding(node.content); + const [e] = await this.llm.embedding(`${node.description}\n\n${update.content}`.trim()); if (e) node.embedding = e.embedding; } + private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> { + return this.llm.ask('', { + model: options.model, + temperature: 0.3, + schema: { + name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true}, + description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true}, + content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true}, + }, + system: `You are a knowledge base editor merging two overlapping Obsidian documents into one. Newer facts win on contradiction. + +Structure loosely: + +# Title +## Summary +## Details +## Related + +Combine both documents, resolve duplication and contradictions. + +Document A ("${a.name}"): +\`\`\`markdown +${stripHeader(a.content)} +\`\`\` + +Document B ("${b.name}"): +\`\`\`markdown +${stripHeader(b.content)} +\`\`\``, + }); + } + private parseFrontmatter(content: string): {fm: Map, body: string} { const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); if (!match) return {fm: new Map(), body: content}; @@ -584,7 +618,7 @@ ${currentBody} } for (const node of touched) { - const [e] = await this.llm.embedding(node.content); + const [e] = await this.llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim()); if (e) node.embedding = e.embedding; this.touch(node.name); }