From 263a65c192aa7580190073dc1c1948cb429c4c5c Mon Sep 17 00:00:00 2001 From: ztimson Date: Sat, 19 Sep 2026 19:27:00 -0400 Subject: [PATCH] Fix open-ai early termination & memory improvements --- src/memory.ts | 526 ++++++++++++++++++++++++++++--------------------- src/open-ai.ts | 53 ++++- 2 files changed, 340 insertions(+), 239 deletions(-) diff --git a/src/memory.ts b/src/memory.ts index 1f42372..664d148 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -2,10 +2,10 @@ import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts'; import {LLMRequest, LLMMessage} from './llm.ts'; import {AiTool} from './tools.ts'; import {KDTree} from './kd-tree.ts'; -import {escapeRegex} from '@ztimson/utils'; -const MERGE_THRESHOLD = 0.12; +const FACT_SIMILARITY_THRESHOLD = 0.62; const PENDING_HEADING = '## Pending'; +const TODO_HEADING = '## Todo list'; const TREE_TOMBSTONE_LIMIT = 0.25; const ALIAS_MATCH_THRESHOLD = 0.55; @@ -13,11 +13,8 @@ export type Memory = { name: string; description: string; content: string; - /** Description embedding — indexed in the KD tree, used for merge/ANN candidate lookup */ embedding: number[]; - /** Title-only embedding, weighted heaviest during recall ranking */ titleEmbedding?: number[]; - /** Chunked body embeddings, best-chunk match used during recall ranking */ bodyEmbeddings?: number[][]; links: string[]; backlinks: string[]; @@ -26,7 +23,6 @@ export type Memory = { type MemoryRef = { name: string; description: string; - /** Cosine distance from the query, present when returned from a search */ distance?: number; } @@ -35,24 +31,32 @@ type FactBucket = { facts: string[]; } +type MemoryTask = { + /** Exact node name / new persistent entity path this task belongs to, or '' for a personal task with no entity (goes to the journal) */ + subject: string; + task: string; + done: boolean; +} + type FactAgentResult = { buckets: FactBucket[]; journal: string; + tasks: MemoryTask[]; } function dedupeFacts(facts: string[]): string[] { const seen = new Map(); - for (const f of facts) { + for(const f of facts) { const clean = f.trim(); - if (clean) seen.set(clean.toLowerCase(), clean); + if(clean) seen.set(clean.toLowerCase(), clean); } return [...seen.values()]; } function cosineDistance(a: number[], b: number[]): number { let dot = 0, normA = 0, normB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; + for(let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; normA += a[i] * a[i]; normB += b[i] * b[i]; } @@ -68,14 +72,13 @@ function cosineSearch(query: number[], memories: Memory[], limit: number): Memor .slice(0, limit); } -/** Re-embed a node's title / description / body fields. Description embedding stays the KD-tree index key. */ async function embedMemoryFields(node: Memory, llm: any): Promise { const body = stripHeader(node.content); const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name); const [descE] = await llm.embedding(node.description || ''); const bodyChunks = body ? await llm.embedding(body) : []; - if (titleE) node.titleEmbedding = titleE.embedding; - if (descE) node.embedding = descE.embedding; + if(titleE) node.titleEmbedding = titleE.embedding; + if(descE) node.embedding = descE.embedding; node.bodyEmbeddings = bodyChunks.map((c: any) => c.embedding).filter(Boolean); } @@ -83,9 +86,14 @@ export function stripHeader(content: string): string { return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart(); } +/** True if a task has no persistent entity of its own and belongs in the journal instead. */ +function isPersonalTask(t: MemoryTask): boolean { + const s = (t.subject ?? '').trim().toLowerCase(); + return !s || s === 'journal' || s.startsWith('journal/'); +} + export class MemoryCache { private tree!: KDTree; - /** Tracks which memories are currently indexed in the tree, keyed by name -> embedding reference */ private indexed = new Map(); public memories: Memory[]; public nodes: MemoryNode[] = []; @@ -98,31 +106,30 @@ export class MemoryCache { this.rebuild(); } - /** Incrementally sync the KD tree against `this.memories` instead of rebuilding from scratch */ private syncTree(): void { const current = new Set(this.memories.map(m => m.name)); - for (const [name, emb] of [...this.indexed]) { + for(const [name, emb] of [...this.indexed]) { const mem = this.memories.find(m => m.name === name); - if (!mem || !current.has(name) || mem.embedding !== emb) { + if(!mem || !current.has(name) || mem.embedding !== emb) { this.tree.remove(p => p.name === name); this.indexed.delete(name); } } - for (const mem of this.memories) { - if (!mem.embedding?.length || this.indexed.has(mem.name)) continue; - if (this.tree.dims === 0) this.tree = new KDTree(mem.embedding.length, 'cosine'); - if (mem.embedding.length !== this.tree.dims) continue; // guard against embedding model/dim drift + for(const mem of this.memories) { + if(!mem.embedding?.length || this.indexed.has(mem.name)) continue; + if(this.tree.dims === 0) this.tree = new KDTree(mem.embedding.length, 'cosine'); + if(mem.embedding.length !== this.tree.dims) continue; // guard against embedding model/dim drift this.tree.insert({vector: mem.embedding, payload: {name: mem.name, description: mem.description}}); this.indexed.set(mem.name, mem.embedding); } - if (this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance(); + if(this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance(); } search(query: number[], limit: number): MemoryRef[] { - if (!this.tree || this.tree.dims === 0) return []; + if(!this.tree || this.tree.dims === 0) return []; return this.tree.knn(query, limit).map(r => ({...r.point.payload, distance: r.distance})); } @@ -133,14 +140,14 @@ export class MemoryCache { update(memory: Memory): void { const existing = this.memories.find(m => m.name === memory.name); - if (existing) Object.assign(existing, memory); + if(existing) Object.assign(existing, memory); else this.memories.push(memory); this.rebuild([existing ?? memory]); } remove(name: string): void { const idx = this.memories.findIndex(m => m.name === name); - if (idx !== -1) { + if(idx !== -1) { this.memories.splice(idx, 1); this.rebuild(); } @@ -168,7 +175,7 @@ class MemoryAccessor { } commit(changed?: Memory[]): MemoryNode[] { - if (this.cache) { + if(this.cache) { this.cache.rebuild(changed); return this.cache.nodes; } @@ -180,14 +187,13 @@ class MemoryAccessor { return nodes.filter(n => n.missing).map(n => n.name); } - /** Cache path uses the KD tree's knn(); raw-array path (no cache available) falls back to a linear cosine scan */ search(vector: number[], limit: number): MemoryRef[] { return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit); } forget(name: string): boolean { const idx = this.list.findIndex(m => m.name === name); - if (idx === -1) return false; + if(idx === -1) return false; this.list.splice(idx, 1); this.commit(); return true; @@ -195,7 +201,7 @@ class MemoryAccessor { async backfillEmbeddings(llm: any): Promise { const missing = this.list.filter(m => !m.embedding?.length); - if (!missing.length) return 0; + if(!missing.length) return 0; await Promise.all(missing.map(node => embedMemoryFields(node, llm))); this.commit(); return missing.length; @@ -241,11 +247,11 @@ export class MemoryManager { name: 'memory_recall', description: 'Read the full content of a memory document', args: { - name: {type: 'string', description: 'Exact memory name', required: true}, + name: {type: 'string', description: 'Exact memory name', required: true} }, fn: (args: any) => { - const mem = this.access(memories).find(args.name); - if (!mem) return 'Document not found'; + const mem = new MemoryAccessor(memories).find(args.name); + if(!mem) return 'Document not found'; this.touch(mem.name); return mem.content; }, @@ -259,7 +265,7 @@ export class MemoryManager { limit: {type: 'number', description: 'Number of memories to return', default: 1}, }, fn: async ({query, limit}) => { - const mem = await this.recollect(query, memories, limit) + const mem = await this.recollect(query, memories, limit); return mem.map(m => `Memory: ${m.name} Description: ${m.description} Links: ${[...m.links, ...m.backlinks].join(', ')} @@ -273,17 +279,16 @@ ${m.content} constructor(private llm: any) {} static normalize(m?: Memory[] | MemoryCache | MemoryOptions) { - if (!m) return null; + if(!m) return null; const raw = m instanceof MemoryCache || Array.isArray(m); return raw ? {memory: m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m}; } - private access(memories: Memory[] | MemoryCache): MemoryAccessor { - return new MemoryAccessor(memories); - } - private stage(node: Memory, block: string): void { - this.ensureDoc(node); + if(!node.content) { + const title = node.name.split('/').pop() ?? node.name; + node.content = this.touchHeader(node, `# ${title}\n`); + } const body = stripHeader(node.content); const idx = body.indexOf(PENDING_HEADING); const newBody = idx === -1 @@ -292,49 +297,28 @@ ${m.content} node.content = this.touchHeader(node, newBody); } - private ensureDoc(node: Memory): void { - if (node.content) return; - const title = node.name.split('/').pop() ?? node.name; - node.content = this.touchHeader(node, `# ${title}\n`); - } - - private sanitizeDescription(text: string): string { - return (text ?? '').replace(/\s+/g, ' ').trim().slice(0, 240); - } - - private relink(memories: Memory[], from: string, to: string): void { - const pattern = new RegExp(`\\[\\[${escapeRegex(from)}\\]\\]`, 'g'); - for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`); - } - - private normalizeLeaf(name: string): string { - return name.trim().toLowerCase().replace(/\s+/g, ' '); - } - - /** - * Resolve a fact-agent proposed subject to an existing node when it's an alias/rename of one. - * Exact match is checked first (cheap, and covers the common case since node names are - * already normalized at creation time). Only falls through to fuzzy alias matching against - * same-root candidates when there's no existing hit — i.e. only on likely-new-doc creation. - */ private resolveSubject(subject: string, store: MemoryAccessor): string { + function normalize(name: string): string { + return name.trim().toLowerCase().replace(/\s+/g, ' '); + } + const trimmed = subject.trim(); const exact = store.find(trimmed); - if (exact) return exact.name; + if(exact) return exact.name; - const normalized = this.normalizeLeaf(trimmed); - const caseInsensitive = store.list.find(m => this.normalizeLeaf(m.name) === normalized); - if (caseInsensitive) return caseInsensitive.name; + const normalized = normalize(trimmed); + const caseInsensitive = store.list.find(m => normalize(m.name) === normalized); + if(caseInsensitive) return caseInsensitive.name; const root = trimmed.split('/')[0]; const leaf = trimmed.split('/').slice(1).join('/') || trimmed; const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed); - if (!candidates.length) return trimmed; + if(!candidates.length) return trimmed; const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name); const probe = leaves.length > 1 ? leaves : [...leaves, '']; const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe); - if (max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name; + if(max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name; return trimmed; } @@ -345,32 +329,48 @@ ${m.content} const response = await this.llm.ask(conversation, { model: options.model, temperature: 0.2, - system: `Extract durable memory from this conversation + system: `Turn this conversation into a persistent memory file by extracting information into organized bullet points -1. Journal recap -- Brief "Captain's Log" of what happened, including useful context, decisions, or events -- Leave empty for trivial exchanges +Think of this like an Obsidian vault with a clear division of responsibility: +- The JOURNAL is a timeline. It answers "what happened, and when" and is the only place with a sense of time. +- ENTITY DOSSIERS are a wiki. They answer "what is currently true about this subject", with no sense of time — only current state. +- Never blur the two: a one-off event, conversation, or debugging session is a journal entry, not an entity, even if it's detailed. -2. Fact buckets -- Extract only durable facts explicitly stated by the USER +1. Journal Log +- A chronological, skimmable log of what actually happened: real discussions, decisions made, progress on projects, problems worked through +- This is NOT a transcript, and it is NOT a step-by-step record, its a compressed log of notable events & developments +- One line per development is usually enough: what was worked on and the outcome, not the blow-by-blow of how +- Skip small talk and trivial exchanges entirely. Skip anything that's purely a todo item (goes in Todo Tasks) or a durable fact about a subject (goes in Entity Dossiers) + +2. Todo Tasks +- Extract concrete tasks the user says need to be done, should be done, or were completed +- Return the task text and whether it is still todo or is done +- A completed task should be marked done, not recreated as a new todo +- Only extract actionable tasks, not general goals or observations +- Assign each task a subject: + - If the task belongs to a persistent entity (a project, a class, etc.), use that entity's exact node name, or a new entity path if it doesn't exist yet + - If it's a personal/life task with no entity of its own (reach out to someone, reply to an email, pay a bill, etc.), leave subject as an empty string — it belongs in the journal, not a new document + +3. Entity Dossiers +- Detailed dossiers with all information regarding a subject - Record the final/end state, not intermediate changes -- Do not extract assistant claims, guesses, greetings, or temporary conversation details +- Ignore assistant claims, guesses, greetings, or temporary details +- NEVER create a document for something that's only meaningful as a point in time — a single conversation, a one-off decision, a debugging session, a date. That's a journal entry, not an entity +- identify its HOME ENTITY: + - The HOME ENTITY name should always be a [abstract|pro]noun + - The grammatical subject/owner of the fact is the strongest clue + - Always preference an existing entity over creating a new one + - New child entities are appropriate only when they are themselves distinct persistent entities + - A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment + - Put project facts under the project they belong to, person facts under the person, etc -For each fact, identify its HOME ENTITY: -- The HOME ENTITY name should always be a [abstract|pro]noun -- The grammatical subject/owner of the fact is the strongest clue -- Prefer an existing entity over creating a new one -- A document represents a persistent entity, not a topic, feature, bug, event, decision, setting, or conversation fragment -- Put project facts under the project they belong to, person facts under the person, etc -- New child entities are appropriate only when they are themselves distinct persistent entities - -Example Paths: +Example Entity Naming Convention: - Projects/[Name] - People/[Name] - History/[Name] - Science/[Name] - [Subject]/[Name] -- Class/[Name]/[Child] +- Class/[Name]/[Chapter] Use [[WikiLinks]] to express relationships between entities. NEVER create documents just to hold relationships Keep journal material in the journal; don't turn journal events into entities unless they represent something persistent @@ -380,7 +380,17 @@ ${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n' ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, schema: { journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false}, - buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: { + tasks: { + type: 'array', description: 'Concrete tasks mentioned or completed in the conversation.', required: false, items: { + type: 'object', items: { + subject: {type: 'string', description: 'Exact node name / new persistent entity path this task belongs to, or an empty string if this is a personal task with no entity of its own (those go in the journal)', required: true}, + task: {type: 'string', description: 'Concise actionable task', required: true}, + done: {type: 'boolean', description: 'Whether the task is completed', required: true}, + }, + } + }, + buckets: { + type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: { type: 'object', items: { subject: {type: 'string', description: 'Exact node name or new persistent entity path', required: true}, facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}}, @@ -391,7 +401,7 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, }); const buckets = new Map(); - for (const bucket of response.buckets ?? []) { + for(const bucket of response.buckets ?? []) { const subject = bucket.subject.trim(); const facts = buckets.get(subject) ?? []; facts.push(...dedupeFacts(bucket.facts)); @@ -401,10 +411,11 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, return { buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})), journal: (response.journal ?? '').trim(), + tasks: response.tasks ?? [], }; } - private getWeekMonday(date: Date = new Date()): string { + private getWeekStart(date: Date = new Date()): string { const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate())); const day = d.getUTCDay(); const diff = day === 0 ? -6 : 1 - day; @@ -412,44 +423,92 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, return d.toISOString().slice(0, 10); } + private journalDescription(journalName?: string): string { + const start = journalName?.split('/').pop() || this.getWeekStart(); + const d = new Date(`${start}T00:00:00Z`); + d.setUTCDate(d.getUTCDate() + 6); + const end = d.toISOString().slice(0, 10); + return `Log from ${start} - ${end}`; + } + + private getIncompleteTodos(content: string): string[] { + const body = stripHeader(content); + const match = body.match(/## Todo list\n([\s\S]*?)(?=\n## |$)/i); + if(!match) return []; + return match[1].split('\n') + .map(line => line.match(/^\s*-\s*\[([ xX])\]\s+(.+?)\s*$/)) + .filter((m): m is RegExpMatchArray => !!m && m[1].toLowerCase() !== 'x') + .map(m => m[2].trim()); + } + private listNodes(memories: Memory[]): MemoryRef[] { return memories.map(m => ({name: m.name, description: m.description})); } - /** Finds the closest merge candidate via the KD tree's knn() instead of a manual O(n) cosine scan */ - private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise { - if (!node.embedding?.length || node.name.startsWith('Journal/')) return null; - const store = this.access(memories); + private async mergeAgent(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise { + function factSimilarity(a: Memory, b: Memory): number { + if(!a.bodyEmbeddings?.length || !b.bodyEmbeddings?.length) return 0; + let best = 0; + for(const av of a.bodyEmbeddings) { + for(const bv of b.bodyEmbeddings) best = Math.max(best, 1 - cosineDistance(av, bv)); + } + return best; + } - const candidate = store.search(node.embedding, 5) - .find(r => r.name !== node.name && !r.name.startsWith('Journal/') && r.distance !== undefined && r.distance <= threshold); - if (!candidate) return null; - const closest = store.find(candidate.name); - if (!closest) return null; + if(!node.embedding?.length || node.name.startsWith('Journal/')) return null; + const store = new MemoryAccessor(memories); + const candidates = store.list + .filter(m => m.name !== node.name && !m.name.startsWith('Journal/')) + .filter(m => factSimilarity(node, m) >= FACT_SIMILARITY_THRESHOLD); - const result = await this.mergeAgent(node, closest, options); - const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []}; - merged.content = this.touchHeader(merged, result.content); - await embedMemoryFields(merged, this.llm); + if(!candidates.length) return null; + const closest = candidates.sort((a, b) => factSimilarity(node, b) - factSimilarity(node, a))[0]; + const result = await this.llm.ask('', { + model: options.model, + temperature: 0.3, + schema: { + aContent: {type: 'string', description: 'Updated document A body in markdown, without frontmatter.', required: true}, + bContent: {type: 'string', description: 'Updated document B body in markdown, without frontmatter.', required: true}, + }, + system: `Maintain these two persistent knowledge-base documents like a wiki. - this.relink(store.list, node.name, merged.name); - this.relink(store.list, closest.name, merged.name); +Do NOT merge, rename, or delete either document. Both represent entities that should remain independently addressable. - this.queues.get(closest.name)?.request?.abort?.(); - this.queues.delete(closest.name); +The documents were selected because their facts may overlap. Your job is to reconcile duplicated information and connect the documents: +- Decide which document is the HOME for each duplicated fact. +- Keep the authoritative copy in that home document. +- In the other document, replace the information with a short preamble and [[WikiLink]] to the home entity explaining the relationship. +- If the documents are distinct entities but merely related, keep their distinct facts and add useful [[WikiLinks]] between them. +- Do not delete useful entity-specific facts just because they are similar. +- Do not invent relationships or facts. +- Preserve useful history, technical specifics, structure, and existing [[WikiLinks]]. +- Most current truth wins when facts conflict. +- Keep both documents concise and information-dense. +- No frontmatter, preamble, filler, or AI commentary. - store.forget(node.name); - store.forget(closest.name); - store.list.push(merged); - store.commit(); +Document A ("${node.name}"): +\`\`\`markdown +${stripHeader(node.content)} +\`\`\` - return merged; +Document B ("${closest.name}"): +\`\`\`markdown +${stripHeader(closest.content)} +\`\`\``, + }); + const a = store.find(node.name); + const b = store.find(closest.name); + if(!a || !b || !result?.aContent || !result?.bContent) return null; + a.content = this.touchHeader(a, result.aContent); + b.content = this.touchHeader(b, result.bContent); + await Promise.all([embedMemoryFields(a, this.llm), embedMemoryFields(b, this.llm)]); + return a; } private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise { const key = node.name; const existing = this.queues.get(key); - if (existing) { + if(existing) { existing.dirty = true; existing.request?.abort?.(); return existing.task; @@ -457,19 +516,19 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, const entry = {dirty: false, request: null, task: Promise.resolve()}; this.queues.set(key, entry); - const store = this.access(memories); + const store = new MemoryAccessor(memories); entry.task = (async () => { - let current = node, merged = false; + let current = node; try { do { entry.dirty = false; await this.docAgent(current, store.list, options, entry); - this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options)); + this.mergeLock = this.mergeLock.then(() => this.mergeAgent(current, memories, options)); const result = await this.mergeLock; - if (result) { current = result; merged = true; } - } while (entry.dirty); + if(result) current = result; + } while(entry.dirty); } finally { - store.commit(merged ? undefined : [node]); + store.commit([node]); this.queues.delete(key); } })(); @@ -479,30 +538,32 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise { if(!memories.includes(node)) return; const currentBody = stripHeader(node.content); - let update; - try { - for (let i = 0; i < 2 && !update?.content; i++) { - const request = this.llm.ask(currentBody, { - model: options.model, - temperature: 0.3, - schema: { - description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true}, - content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true}, - }, - system: `You maintain one persistent knowledge-base document + const journal = node.name.startsWith('Journal/'); + const system = (journal + ? `You maintain one persistent journal document -Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished +Rewrite the ENTIRE journal, folding "## Pending" into the existing content removing the heading + +Journal design: +- Preserve the chronological daily log +- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections +- Group information by day under a date heading +- Keep journal entries high level and concise: what was worked on and the outcome, not a step-by-step record of how — that detail lives in conversation history, not here +- Use [[WikiLinks]] for persistent entities; don't turn ordinary journal events into entities +- No frontmatter, preamble, filler, or AI commentary` + : `You maintain one persistent knowledge-base entity document + +Rewrite the ENTIRE document, folding "## Pending" into the existing content. Remove the Pending section when finished. Document design: -- The document represents one entity. Keep information about that entity together +- The document represents one persistent entity. Keep information about that entity together and organized into sections +- Merge any pending information in, newest fact wins conflicts; remove redundant content +- Maintain a single \`## Todo list\` section for this entity: reconcile tasks semantically (merge equivalent tasks, remove duplicates, preserve incomplete tasks, check off completed ones), and keep it distinct from the narrative/fact sections - Let the structure fit the entity; there is NO fixed template -- Preserve useful existing headings and organization. Don't redesign the document without reason - Add headings only when they meaningfully organize recurring information; don't create headings for one-off facts - Keep the document concise and information-dense without removing useful technical specifics -- Current truth wins when facts conflict. Preserve older context only when it adds useful meaning -- Use [[WikiLinks]] for specific related entities; don't create redundant content for linked entities -- Avoid generic filler sections such as Notes, Miscellaneous, Recent, Updates, or Conversation -- No frontmatter, preamble, filler, or AI commentary +- Current truth wins when facts conflict. Preserve older conflict as context, only when it adds useful meaning +- No frontmatter, preamble, filler, or AI commentary`) + ` Available nodes to link to: ${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'} @@ -510,85 +571,55 @@ ${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).jo Current document: \`\`\`markdown ${currentBody} -\`\`\``, +\`\`\``; + let update; + try { + for(let i = 0; i < 2 && !update?.content; i++) { + const request = this.llm.ask(currentBody, { + model: options.model, + temperature: 0.3, + schema: { + description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true}, + content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true}, + }, + system, }); entry.request = request; update = await request; } - } catch (err: any) { - if (err?.name === 'AbortError') return; + } catch(err: any) { + if(err?.name === 'AbortError') return; throw err; } finally { entry.request = null; } - if (!update?.content) return; - node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user'; + if(!update?.content) return; + node.description = node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.name !== 'People/User' ? update.description.replaceAll(/[\n:]/g, '') : 'All information about the current user'; node.content = this.touchHeader(node, update.content); await embedMemoryFields(node, this.llm); } - private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> { - const modifiedOf = (m: Memory) => this.parseFrontmatter(m.content).fm.get('modified') || 'unknown'; - - return this.llm.ask('', { - model: options.model, - temperature: 0.3, - schema: { - name: {type: 'string', description: 'Canonical path for the merged entity', required: true}, - description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true}, - content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true}, - }, - system: `Determine whether these two documents represent the SAME persistent entity. - -Similarity of subject matter is NOT enough. Do not merge documents merely because they discuss the same project, person, technology, topic, or related work. - -Merge only when the evidence indicates they are duplicate identities, aliases, renamed entities, or two documents accidentally created for the same real-world entity. If they are distinct entities, they must remain separate. - -If they are the same entity: -- Choose the canonical/most established path. -- Combine their information into one document and remove duplication. -- Preserve useful structure, technical specifics, history, and [[WikiLinks]]. -- Prefer newer information when facts conflict. -- Return the canonical entity name and the fully reconciled document. - -Document A ("${a.name}", last modified ${modifiedOf(a)}): -\`\`\`markdown -${stripHeader(a.content)} -\`\`\` - -Document B ("${b.name}", last modified ${modifiedOf(b)}): -\`\`\`markdown -${stripHeader(b.content)} -\`\`\``, - }); - } - private parseFrontmatter(content: string): {fm: Map, body: string} { const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); - if (!match) return {fm: new Map(), body: content}; + if(!match) return {fm: new Map(), body: content}; const fm = new Map(); - for (const line of match[1].split('\n')) { + for(const line of match[1].split('\n')) { const i = line.indexOf(':'); - if (i === -1) continue; + if(i === -1) continue; const key = line.slice(0, i).trim(); const raw = line.slice(i + 1).trim(); let value = raw; - try { value = JSON.parse(raw); } catch { /* legacy unquoted value, keep raw */ } + try { value = JSON.parse(raw); } catch { } fm.set(key, value); } return {fm, body: match[2]}; } - /** - * Writes the code-owned frontmatter block. `body` is passed through stripHeader() first so a - * model that ignores instructions and hallucinates its own `---` block can never corrupt or - * duplicate the real frontmatter — the LLM only ever gets to influence the body. - */ private touchHeader(node: Memory, body: string): string { const {fm} = this.parseFrontmatter(node.content); fm.set('name', node.name); - fm.set('description', node.description || ''); + fm.set('description', (node.name.startsWith('Journal/') ? this.journalDescription(node.name) : node.description) || 'Persistent memory document'); fm.set('modified', new Date().toISOString()); return this.writeFrontmatter(fm, stripHeader(body)); } @@ -599,8 +630,8 @@ ${stripHeader(b.content)} } decay() { - for (const [name, ttl] of this.recentlyTouched) { - if (ttl <= 1) this.recentlyTouched.delete(name); + for(const [name, ttl] of this.recentlyTouched) { + if(ttl <= 1) this.recentlyTouched.delete(name); else this.recentlyTouched.set(name, ttl - 1); } } @@ -610,46 +641,43 @@ ${stripHeader(b.content)} } forget(name: string, memories: Memory[] | MemoryCache): boolean { - return this.access(memories).forget(name); - } - - /** Ranks a candidate pool by weighted title/description/body similarity against the query embedding */ - private rankByFields(query: number[], candidates: Memory[], limit: number): Memory[] { - const scored = candidates.map(m => { - const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0; - const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0; - const bodySim = m.bodyEmbeddings?.length - ? Math.max(...m.bodyEmbeddings.map(b => 1 - cosineDistance(query, b))) - : 0; - return {memory: m, score: titleSim * 0.5 + descSim * 0.35 + bodySim * 0.15}; - }); - return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory); + return new MemoryAccessor(memories).forget(name); } async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise { - const store = this.access(memories); - if (!store.list.length) return []; + function rank(query: number[], candidates: Memory[], limit: number): Memory[] { + const scored = candidates.map(m => { + const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0; + const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0; + const bodySim = m.bodyEmbeddings?.length + ? Math.max(...m.bodyEmbeddings.map(b => 1 - cosineDistance(query, b))) + : 0; + return {memory: m, score: titleSim * 0.5 + descSim * 0.35 + bodySim * 0.15}; + }); + return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory); + } + const store = new MemoryAccessor(memories); + if(!store.list.length) return []; await store.backfillEmbeddings(this.llm); const [e] = await this.llm.embedding(query); - if (!e) return []; + if(!e) return []; - // Description embedding is the cheap ANN index key; pull a wider pool then re-rank by field weight const pool = store.search(e.embedding, Math.max(limit * 3, limit)); const poolMemories = pool.map(r => store.find(r.name)).filter((m): m is Memory => !!m); - const ranked = this.rankByFields(e.embedding, poolMemories, limit); + const ranked = rank(e.embedding, poolMemories, limit); const found = new Set(ranked.map(m => m.name)); - if (graphDepth > 0) { + if(graphDepth > 0) { let frontier = [...found]; - for (let depth = 0; depth < graphDepth && frontier.length; depth++) { + for(let depth = 0; depth < graphDepth && frontier.length; depth++) { const next: string[] = []; - for (const name of frontier) { + for(const name of frontier) { const node = store.find(name); - if (!node) continue; - for (const link of node.links) { - if (!found.has(link) && store.find(link)) { + if(!node) continue; + for(const link of node.links) { + if(!found.has(link) && store.find(link)) { found.add(link); next.push(link); } @@ -668,35 +696,75 @@ ${stripHeader(b.content)} const conversation = history .filter(h => h.role === 'user' || h.role === 'assistant') .map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim(); - if (!conversation) return []; + if(!conversation) return []; const uid = `${Date.now()}_${Math.random().toString(36).slice(2)}`; const pending = {role: 'tool', name: 'memory_process', id: uid, content: conversation} as unknown as LLMMessage; history.push(pending); - const store = this.access(memories); - const {buckets, journal} = await this.factAgent(conversation, store, options); + const store = new MemoryAccessor(memories); + const {buckets, journal, tasks} = await this.factAgent(conversation, store, options); const touched: Memory[] = []; - if (journal) { - const journalName = `Journal/${this.getWeekMonday()}`; + const personalTasks = tasks.filter(isPersonalTask); + const entityTasks = tasks.filter(t => !isPersonalTask(t)); + + if(journal || personalTasks.length) { + const journalName = `Journal/${this.getWeekStart()}`; let jnode = store.find(journalName); - if (!jnode) { - jnode = {name: journalName, description: '', content: '', embedding: [], links: [], backlinks: []}; + const isNew = !jnode; + if(!jnode) { + jnode = { + name: journalName, + description: this.journalDescription(), + content: '', + embedding: [], + links: [], + backlinks: [], + }; store.list.push(jnode); } - this.stage(jnode, `### ${new Date().toISOString().slice(0, 10)}\n${journal}`); + + const blocks: string[] = []; + if(journal) blocks.push(`### ${new Date().toISOString().slice(0, 10)}\n${journal}`); + if(isNew) { + const previousDate = new Date(`${this.getWeekStart()}T00:00:00Z`); + previousDate.setUTCDate(previousDate.getUTCDate() - 7); + const previous = store.find(`Journal/${previousDate.toISOString().slice(0, 10)}`); + if(previous) { + const todos = this.getIncompleteTodos(previous.content); + if(todos.length) blocks.push(`${TODO_HEADING}\n${todos.map(task => `- [ ] ${task}`).join('\n')}`); + } + } + if(personalTasks.length) blocks.push(`${TODO_HEADING}\n${personalTasks.map(task => `- [${task.done ? 'x' : ' '}] ${task.task}`).join('\n')}`); + if(blocks.length) this.stage(jnode, blocks.join('\n\n')); touched.push(jnode); } - for (const {subject, facts} of buckets) { + const entityStaging = new Map(); + for(const {subject, facts} of buckets) { const resolved = this.resolveSubject(subject, store); + const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []}; + entry.facts.push(...facts); + entityStaging.set(resolved, entry); + } + for(const task of entityTasks) { + const resolved = this.resolveSubject(task.subject, store); + const entry = entityStaging.get(resolved) ?? {facts: [], tasks: []}; + entry.tasks.push(task); + entityStaging.set(resolved, entry); + } + + for(const [resolved, {facts, tasks: subjectTasks}] of entityStaging) { let node = store.find(resolved); - if (!node) { - node = {name: resolved, description: '', content: '', embedding: [], links: [], backlinks: []}; + if(!node) { + node = {name: resolved, description: 'Persistent memory document', content: '', embedding: [], links: [], backlinks: []}; store.list.push(node); } - this.stage(node, facts.map(f => `- ${f}`).join('\n')); + const blocks: string[] = []; + if(facts.length) blocks.push(facts.map(f => `- ${f}`).join('\n')); + if(subjectTasks.length) blocks.push(`${TODO_HEADING}\n${subjectTasks.map(t => `- [${t.done ? 'x' : ' '}] ${t.task}`).join('\n')}`); + if(blocks.length) this.stage(node, blocks.join('\n\n')); touched.push(node); } @@ -705,7 +773,7 @@ ${stripHeader(b.content)} this.touch(node.name); })); - if (touched.length) { + if(touched.length) { store.commit(touched); (pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`; Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {}))); @@ -718,7 +786,7 @@ ${stripHeader(b.content)} } async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise { - const store = this.access(memories); + const store = new MemoryAccessor(memories); const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING)); await Promise.all(targets.map(node => this.reconcile(node, memories, options))); store.commit(); diff --git a/src/open-ai.ts b/src/open-ai.ts index d23833e..fe6053e 100644 --- a/src/open-ai.ts +++ b/src/open-ai.ts @@ -36,8 +36,10 @@ export class OpenAi extends LLMProvider { private toWire(history: LLMMessage[], system?: string): any[] { const wire: any[] = []; if(system) wire.push({role: 'system', content: system}); + for(let i = 0; i < history.length; i++) { const h = history[i]; + if(h.role !== 'tool') { wire.push({role: h.role, content: this.toWireContent(h.content)}); continue; @@ -45,24 +47,34 @@ export class OpenAi extends LLMProvider { const calls: any[] = []; const results: any[] = []; + while(i < history.length && history[i].role === 'tool') { - const tool = history[i]; + const tool: any = history[i]; + calls.push({ id: tool.id, type: 'function', function: { name: tool.name, - arguments: JSON.stringify(tool.args) + arguments: JSON.stringify(tool.args || {}) } }); + results.push({ role: 'tool', tool_call_id: tool.id, content: tool.error || tool.content || '' }); + i++; } - wire.push({role: 'assistant', content: null, tool_calls: calls}); + + wire.push({ + role: 'assistant', + content: null, + tool_calls: calls + }); + wire.push(...results); i--; } @@ -76,13 +88,12 @@ export class OpenAi extends LLMProvider { if(!options.history) options.history = []; const history = options.history; if(message) history.push({role: 'user', content: message, timestamp: Date.now()}); - const tools = options.tools || this.ai.options.llm?.tools || []; const requestParams: any = { model: options.model || this.model, stream: !!options.stream, - max_completion_tokens: options.maxTokens || this.ai.options.llm?.maxTokens || undefined, - temperature: options.temperature || this.ai.options.llm?.temperature || undefined, + max_completion_tokens: options.maxTokens ?? this.ai.options.llm?.maxTokens, + temperature: options.temperature ?? this.ai.options.llm?.temperature, tools: tools.map(t => ({ type: 'function', function: { @@ -112,7 +123,10 @@ export class OpenAi extends LLMProvider { try { let terminal = false; + let iteration = 0; + do { + iteration++; requestParams.messages = this.toWire(history.filter(h => h.role !== 'system'), options.system); const callStart = Date.now(); @@ -126,36 +140,46 @@ export class OpenAi extends LLMProvider { let usage: any; let finishReason: string | undefined; let msg: any = {content: '', tool_calls: []}; + let streamedChars = 0; + if(options.stream) { let streamCompleted = false; try { for await (const chunk of resp) { if(controller.signal.aborted) break; if(chunk.usage) usage = chunk.usage; + const choice = chunk.choices?.[0]; if(choice?.finish_reason) finishReason = choice.finish_reason; + if(choice?.delta?.content) { msg.content += choice.delta.content; + streamedChars += choice.delta.content.length; options.stream({text: choice.delta.content}); } + if(choice?.delta?.tool_calls) { for(const deltaTC of choice.delta.tool_calls) { const index = deltaTC.index ?? msg.tool_calls.length; let existing = msg.tool_calls.find((tc: any) => tc.index === index); + if(!existing) { existing = {index, id: '', function: {name: '', arguments: ''}}; msg.tool_calls.push(existing); } + if(deltaTC.id) existing.id = deltaTC.id; if(deltaTC.function?.name) existing.function.name = deltaTC.function.name; if(deltaTC.function?.arguments) existing.function.arguments += deltaTC.function.arguments; } } } + streamCompleted = true; } catch(err) { if(!controller.signal.aborted) throw err; } + if(streamCompleted && !finishReason) finishReason = msg.tool_calls.length ? 'tool_calls' : 'stop'; } else { usage = resp.usage; @@ -164,9 +188,7 @@ export class OpenAi extends LLMProvider { } const duration = Date.now() - callStart; - const tps = usage?.completion_tokens && duration > 0 - ? usage.completion_tokens / (duration / 1000) - : 0; + const tps = usage?.completion_tokens && duration > 0 ? usage.completion_tokens / (duration / 1000) : 0; if(finishReason === 'length' && !controller.signal.aborted) { if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps}); @@ -178,10 +200,20 @@ export class OpenAi extends LLMProvider { } const toolCalls = msg.tool_calls || []; + if(toolCalls.length && !controller.signal.aborted) { if(msg.content?.trim()) history.push({role: 'assistant', content: msg.content.trim(), timestamp: Date.now(), duration, tps}); + const entries = toolCalls.map((tc: any) => { - const entry: any = {role: 'tool', id: tc.id, name: tc.function.name, args: JSONAttemptParse(tc.function.arguments, {}), content: undefined, timestamp: Date.now()}; + const entry: any = { + role: 'tool', + id: tc.id, + name: tc.function.name, + args: JSONAttemptParse(tc.function.arguments, {}), + content: undefined, + timestamp: Date.now() + }; + history.push(entry); return {tc, entry}; }); @@ -195,6 +227,7 @@ export class OpenAi extends LLMProvider { if(chunk.done) return; options.stream!(chunk); }); + const result = await tool.fn(entry.args, toolStream, this.ai, tc.id); entry.content = typeof result === 'object' ? JSONSanitize(result) : result; } catch(err: any) {