From 9c04e58c633fe7c76f10b3422d32dbded7e5dd56 Mon Sep 17 00:00:00 2001 From: ztimson Date: Tue, 4 Aug 2026 12:24:23 -0400 Subject: [PATCH] Pass deligate subagents full history, improved memory managment --- src/llm.ts | 48 +---- src/memory.ts | 469 ++++++++++++++++++------------------------- tests/llm.spec.ts | 167 +++++++++++++++ tests/memory.spec.ts | 256 +++++++++++++++++++++++ 4 files changed, 631 insertions(+), 309 deletions(-) create mode 100644 tests/llm.spec.ts create mode 100644 tests/memory.spec.ts diff --git a/src/llm.ts b/src/llm.ts index 95aa258..52080fa 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -122,27 +122,25 @@ class LLM { this.memoryManager = new MemoryManager(this); } - private setupAgent(agents: Agent[] = [], allAgents: Agent[], pending: Map, aborts: (() => void)[], depth = 0): AiTool[] { + private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: (() => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] { return agents.map(a => { const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`; return { name: toolName, description: `${a.delegate ? 'Delegate to ' : ''}Subagent: ${a.description || a.name}`, - args: { + args: (a.delegate ? {} : { context: {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true}, instructions: {type: 'string', description: 'Detailed instructions for subagent to complete', required: true}, - }, + }), fn: async (args: any, stream: any, ai: any, id?: string) => { if(depth >= MAX_AGENT_DEPTH) return 'Max agent delegation depth exceeded'; - const subHistory: LLMMessage[] = []; // Opt-in only, self always excluded regardless of whitelist const nested = (a.agents || []) .map(name => allAgents.find(x => x.name === name)) .filter((x): x is Agent => !!x && x.name !== a.name); - const start = Date.now(); - const request = this.ask(`${args.instructions}${args.context ? `\n\n${args.context}` : ''}`, { + const request = this.ask(a.delegate ? '' : `${args.instructions}${args.context ? `\n\n${args.context}` : ''}`, { system: `You are a specialized subagent. ${a.delegate ? 'Your output streams directly to the user for the remainder of this turn. You are mid conversation - dispense with greetings.' : 'You are wrapped in a tool call that will be analysis by an LLM - dispense with conversation'} As a subagent, focus on executing your task completely using available tools and returning only the final result - no commentary, questions, or dialogue. @@ -150,7 +148,7 @@ ${a.system}`, model: a.model || undefined, temperature: a.temperature, stream: a.delegate ? stream : undefined, - history: subHistory, + history: a.delegate ? history : [], mcp: a.mcp || undefined, skills: a.skills || undefined, tools: a.tools || undefined, @@ -159,14 +157,9 @@ ${a.system}`, } as any); aborts.push(request.abort); const resp = await request; - const duration = Date.now() - start; - const assistantTurns = subHistory.filter((h: any) => h.role === 'assistant' && h.duration); - const genTime = assistantTurns.reduce((s, h: any) => s + h.duration, 0); - const genTokens = assistantTurns.reduce((s, h: any) => s + (h.tps || 0) * (h.duration / 1000), 0); - const tps = genTime > 0 ? genTokens / (genTime / 1000) : 0; if(a.delegate) { - pending.set(id, {resp, subHistory, duration, tps}); + delegateState.resp = resp; return ''; } return resp; @@ -292,8 +285,8 @@ ${a.system}`, // Agents const agents = options.agents || this.ai.options?.llm?.agents; - const pendingDelegates = new Map(); - if(agents?.length) tools.push(...this.setupAgent(agents, agents, pendingDelegates, nestedAborts, options._agentDepth || 0)); + const delegateState: {resp: string | null} = {resp: null}; + if(agents?.length) tools.push(...this.setupAgent(agents, agents, history, nestedAborts, options._agentDepth || 0, delegateState)); // Memory const mem = MemoryManager.normalize(options.memory); @@ -335,7 +328,6 @@ Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.nam if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'}); - // Time each tool call's real execution so its history entry gets its own duration/tps const toolTimings = new Map(); tools = this.wrapToolTiming(tools, toolTimings); @@ -345,34 +337,14 @@ Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.nam request = this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')}); let resp = await request; - // Providers stamp duration/tps on assistant entries themselves (from real API usage). - // Overwrite tool entries with actual tool-execution timing instead of the LLM call timing. + // Capture meta (duration / tps) for(const h of history) { if(h.role === 'tool' && toolTimings.has(h.id)) Object.assign(h, toolTimings.get(h.id)); } - // Spice delegated agents response into history - let lastDelegateResp: string | null = null; - if(pendingDelegates.size) { - for(let i = 0; i < history.length; i++) { - const h: any = history[i]; - if(h.role !== 'tool' || !pendingDelegates.has(h.id)) continue; - const {resp: delegateResp, subHistory, duration, tps} = pendingDelegates.get(h.id)!; - pendingDelegates.delete(h.id); - const insert: LLMMessage[] = [...subHistory.filter(sh => sh.role === 'tool'), {role: 'assistant', content: delegateResp, timestamp: Date.now(), duration, tps}]; - history.splice(i + 1, 0, ...insert); - lastDelegateResp = delegateResp; - i += insert.length; - } - } + if(typeof resp === 'string' && !resp.trim() && delegateState.resp !== null) resp = delegateState.resp; - // If the orchestrator added no commentary of its own, its answer IS the delegate's answer - if(typeof resp === 'string' && !resp.trim() && lastDelegateResp !== null) resp = lastDelegateResp; - - // Trim memory injections from history if(mem?.tool) history.splice(0, history.length, ...history.filter(h => h.role !== 'tool' || h.name !== 'memory_recall')); - - // Auto-memorize before compressing if(options.compress && this.estimateTokens(history) >= options.compress.max) { if(mem?.update) await this.memoryManager.memorize(history, mem.memory, {model: options.memoryModel || this.defaultModel, ...options}); const compressed = await this.compressHistory(history, options.compress.max, options.compress.min, options); diff --git a/src/memory.ts b/src/memory.ts index a5b94d5..9101cf9 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -2,6 +2,16 @@ import {LLMRequest, LLMMessage} from './llm.ts'; import {AiTool} from './tools.ts'; import {KDPoint, KDTree} from './kd-tree.ts'; +const FACTS_HEADING = '## Facts'; + +const GENERIC_TEMPLATE = `# {{Title}} + +## Summary + +## Details + +## Related`; + export class MemoryCache { private tree: KDTree; public memories: Memory[]; @@ -77,46 +87,35 @@ export type Memory = { description: string; content: string; embedding: number[]; -} - -export type MemoryRef = { - name: string; - description: string; -} - -export type FactBucket = { - subject: string; - facts: string[]; -} - -export type MemoryNode = { - name: string; - missing: boolean; links: string[]; backlinks: string[]; } +type MemoryRef = { + name: string; + description: string; +} + +type FactBucket = { + subject: string; + facts: string[]; +} + function extractLinks(content: string): string[] { - if(!content) return []; + if (!content) return []; const matches = content.matchAll(/\[\[([^\]]+)\]\]/g); return [...new Set([...matches].map(m => m[1].trim()))]; } -export function extractMetadata(content: string): {links: string[], backlinks: string[]} { - const match = content.match(/^---\n([\s\S]*?)\n---/); - if (!match) return {links: [], backlinks: []}; - - const fm = match[1]; - const getList = (key: string): string[] => { - const m = fm.match(new RegExp(`^${key}:\\s*\\[(.*)\\]$`, 'm')); - if (!m || !m[1].trim()) return []; - return m[1].split(',').map(s => s.trim().replace(/^"|"$/g, '')).filter(Boolean); - }; - - return { - links: getList('links'), - backlinks: getList('backlinks'), - }; +export function rebuildGraph(memories: Memory[]): void { + for (const m of memories) m.links = extractLinks(m.content).filter(l => l !== m.name); + for (const m of memories) m.backlinks = []; + for (const m of memories) { + for (const link of m.links) { + const target = memories.find(t => t.name === link); + if (target) target.backlinks.push(m.name); + } + } } function dedupeFacts(facts: string[]): string[] { @@ -147,23 +146,11 @@ function getWeekMonday(date: Date = new Date()): string { return d.toISOString().slice(0, 10); } -function getWeekSunday(monday: string): string { - const d = new Date(`${monday}T00:00:00Z`); - d.setUTCDate(d.getUTCDate() + 6); - return d.toISOString().slice(0, 10); -} - export class MemoryManager { private recentlyTouched = new Map(); - private pendingMemorizations = new Map(); - private queues = new Map void} | null, task: Promise, }>(); @@ -176,7 +163,7 @@ export class MemoryManager { name: {type: 'string', description: 'Exact memory name', required: true}, }, fn: (args: any) => { - const mems = memories instanceof MemoryCache ? memories.memories : memories; + const mems = this.unwrap(memories); const mem = mems.find(m => m.name === args.name); if (!mem) return 'Document not found'; this.touch(mem.name); @@ -205,110 +192,58 @@ export class MemoryManager { return raw ? {memory: m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m}; } - private async createTempMemory(conversation: string): Promise { - const timestamp = Date.now(); - const content = `--- -name: _temp_${timestamp} -description: Temporary memory - processing in background -tags: [_temporary] -links: [] -backlinks: [] -modified: ${new Date().toISOString()} ---- - -# Recent Conversation (Processing) - -${conversation}`; - const [e] = await this.llm.embedding(content); - return { - name: `_temp_${timestamp}`, - description: 'Temporary memory - processing in background', - content, - embedding: e?.embedding || [], - }; + private unwrap(memories: Memory[] | MemoryCache): Memory[] { + return memories instanceof MemoryCache ? memories.memories : memories; } - private applyHeader(content: string, header: string): string { - return `${header}\n\n${this.stripHeader(content)}`; + private sync(memories: Memory[] | MemoryCache): void { + if (memories instanceof MemoryCache) memories.rebuild(); } - private async backgroundMemorization(conversation: string, memories: Memory[] | MemoryCache, options: LLMRequest, tempName: string): Promise { - const mem = memories instanceof MemoryCache ? memories.memories : memories; - const monday = getWeekMonday(); - const sunday = getWeekSunday(monday); - const buckets = await this.factAgent(conversation, mem, options, monday); - if(!buckets.length) return; - const jobs = [...buckets].map(({subject, facts}) => { - let node = mem.find(m => m.name === subject); - if(!node) { - node = {name: subject, description: '', content: '', embedding: [],}; - mem.push(node); - } - const week = subject.startsWith('Journal/') ? {monday, sunday} : undefined; - return this.enqueue(node, facts, mem, options, tempName, week); - }); - await Promise.all(jobs); - } - - private buildHeader(node: Memory, week?: {monday: string, sunday: string}, links: string[] = [], backlinks: string[] = []): string { - const tags = node.name.split('/')[0]?.toLowerCase(); - const lines = [ - '---', - `name: ${node.name}`, - `description: ${node.description || ''}`, - tags ? `tags: [${tags}]` : '', - links.length ? `links: [${links.map(l => `"${l}"`).join(', ')}]` : 'links: []', - backlinks.length ? `backlinks: [${backlinks.map(l => `"${l}"`).join(', ')}]` : 'backlinks: []', - week ? `week: ${week.monday} – ${week.sunday}` : '', - `modified: ${new Date().toISOString()}`, - '---', - ].filter(Boolean); - return lines.join('\n'); - } - - private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] { - const scored = memories - .filter(m => m.embedding?.length) - .map(m => ({ - ref: {name: m.name, description: m.description}, - distance: cosineDistance(query, m.embedding), - })) - .sort((a, b) => a.distance - b.distance) - .slice(0, limit); - return scored.map(s => s.ref); - } - - /** - * Coalescing queue: if a doc is already compiling, abort the in-flight run, merge its - * facts with the new ones and restart. Never blocks a pending update, never drops facts. - */ - private enqueue(node: Memory, facts: string[], memories: Memory[] | MemoryCache, options: LLMRequest, tempName: string, week?: {monday: string, sunday: string}): Promise { - const key = node.name; - const existing = this.queues.get(key); - if (existing) { - existing.pending.push(...facts); - existing.request?.abort?.(); - return existing.task; + private parseFrontmatter(content: string): {fm: Map, body: string} { + const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); + if (!match) return {fm: new Map(), body: content}; + const fm = new Map(); + for (const line of match[1].split('\n')) { + const i = line.indexOf(':'); + if (i === -1) continue; + fm.set(line.slice(0, i).trim(), line.slice(i + 1).trim()); } - - const entry: {pending: string[], request: {abort?: () => void} | null, task: Promise} = {pending: [...facts], request: null, task: Promise.resolve()}; - this.queues.set(key, entry); - const m = memories instanceof MemoryCache ? memories.memories : memories; - entry.task = (async () => { - while (entry.pending.length) { - const batch = dedupeFacts(entry.pending.splice(0, entry.pending.length)); - const written = await this.docAgent(node, batch, m, options, tempName, week, entry); - if (!written) entry.pending.unshift(...batch); - } - })().finally(() => { - this.queues.delete(key); - if(!this.queues.size && memories instanceof MemoryCache) memories.rebuild(); - }); - return entry.task; + return {fm, body: match[2]}; } - private listNodes(memories: Memory[]): MemoryRef[] { - return memories.map(m => ({name: m.name, description: m.description})); + private writeFrontmatter(fm: Map, body: string): string { + const lines = [...fm.entries()].map(([k, v]) => `${k}: ${v}`); + return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`; + } + + private stripHeader(content: string): string { + return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart(); + } + + private touchHeader(node: Memory, body: string): string { + const {fm} = this.parseFrontmatter(node.content); + fm.set('name', node.name); + fm.set('description', node.description || ''); + fm.set('modified', new Date().toISOString()); + return this.writeFrontmatter(fm, body); + } + + private ensureDoc(node: Memory): void { + if (node.content) return; + const title = node.name.split('/').pop() ?? node.name; + node.content = this.touchHeader(node, `# ${title}\n`); + } + + private appendFacts(node: Memory, facts: string[]): void { + this.ensureDoc(node); + const body = this.stripHeader(node.content); + const bullets = facts.map(f => `- ${f}`).join('\n'); + const idx = body.indexOf(FACTS_HEADING); + const newBody = idx === -1 + ? `${body.trimEnd()}\n\n${FACTS_HEADING}\n${bullets}\n` + : `${body.slice(0, idx + FACTS_HEADING.length)}\n${bullets}${body.slice(idx + FACTS_HEADING.length)}`; + node.content = this.touchHeader(node, newBody); } decay() { @@ -318,71 +253,27 @@ ${conversation}`; } } - forget(name: string, memories: Memory[] | MemoryCache): boolean { - const mem = memories instanceof MemoryCache ? memories.memories : memories; - const idx = mem.findIndex(m => m.name === name); - if (idx === -1) return false; - - for (const node of mem) { - const {links, backlinks} = extractMetadata(node.content); - const newBacklinks = backlinks.filter(b => b !== name); - const newLinks = links.filter(l => l !== name); - - if (newBacklinks.length !== backlinks.length || newLinks.length !== links.length) { - node.content = this.updateFrontmatter(node.content, { - links: newLinks, - backlinks: newBacklinks, - }); - } - } - - mem.splice(idx, 1); - - if (memories instanceof MemoryCache) memories.rebuild(); - return true; + touch(name: string, ttl = 2) { + this.recentlyTouched.set(name, ttl); } getTouched(): string[] { return [...this.recentlyTouched.keys()]; } - async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise { - const conversation = history - .filter(h => h.role === 'user' || h.role === 'assistant') - .map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim(); - if(!conversation) return []; + forget(name: string, memories: Memory[] | MemoryCache): boolean { + const mem = this.unwrap(memories); + const idx = mem.findIndex(m => m.name === name); + if (idx === -1) return false; - const trackingId = `${Date.now()}_${Math.random()}`; - const tempMemory = await this.createTempMemory(conversation); - const mem = memories instanceof MemoryCache ? memories.memories : memories; - mem.push(tempMemory); - if (memories instanceof MemoryCache) memories.rebuild(); - this.pendingMemorizations.set(trackingId, { - memories, - tempMemoryName: tempMemory.name, - timestamp: Date.now(), - }); - - try { - await this.backgroundMemorization(conversation, memories, options, tempMemory.name); - const finalMem = memories instanceof MemoryCache ? memories.memories : memories; - return finalMem.filter(m => !m.name.startsWith('_temp_')); - } finally { - const pending = this.pendingMemorizations.get(trackingId); - if (pending) { - const cleanMem = pending.memories instanceof MemoryCache - ? pending.memories.memories - : pending.memories; - const idx = cleanMem.findIndex(m => m.name === pending.tempMemoryName); - if (idx !== -1) cleanMem.splice(idx, 1); - if (pending.memories instanceof MemoryCache) pending.memories.rebuild(); - } - this.pendingMemorizations.delete(trackingId); - } + mem.splice(idx, 1); + rebuildGraph(mem); + this.sync(memories); + return true; } async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise { - const mem: Memory[] = memories instanceof MemoryCache ? memories.memories : memories; + const mem = this.unwrap(memories); if (!mem.length) return []; const [e] = await this.llm.embedding(query); @@ -400,8 +291,7 @@ ${conversation}`; for (const name of frontier) { const node = mem.find(m => m.name === name); if (!node) continue; - const {links} = extractMetadata(node.content); - for (const link of links) { + for (const link of node.links) { if (!found.has(link) && mem.find(m => m.name === link)) { found.add(link); next.push(link); @@ -419,114 +309,151 @@ ${conversation}`; return ordered.map(n => mem.find(m => m.name === n)!).filter(Boolean); } - touch(name: string, ttl = 2) { - this.recentlyTouched.set(name, ttl); + private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] { + const scored = memories + .filter(m => m.embedding?.length) + .map(m => ({ + ref: {name: m.name, description: m.description}, + distance: cosineDistance(query, m.embedding), + })) + .sort((a, b) => a.distance - b.distance) + .slice(0, limit); + return scored.map(s => s.ref); } - private updateFrontmatter(content: string, updates: {links?: string[], backlinks?: string[]}): string { - const match = content.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/); - if (!match) return content; + private listNodes(memories: Memory[]): MemoryRef[] { + return memories.map(m => ({name: m.name, description: m.description})); + } - const [, fm, body] = match; - let newFm = fm; + async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise { + const conversation = history + .filter(h => h.role === 'user' || h.role === 'assistant') + .map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim(); + if (!conversation) return []; - if (updates.links !== undefined) { - const linksList = updates.links.length ? `[${updates.links.map(l => `"${l}"`).join(', ')}]` : '[]'; - newFm = newFm.replace(/^links:.*$/m, `links: ${linksList}`); + const uid = `${Date.now()}_${Math.random().toString(36).slice(2)}`; + // NOTE: adjust field names below (id/tool_call_id/name) to match your LLMMessage/tool-call schema. + const pending = {role: 'tool', name: 'memory_process', id: uid, content: 'Processing…'} as unknown as LLMMessage; + history.push(pending); + + const mem = this.unwrap(memories); + const buckets = await this.factAgent(conversation, mem, options, getWeekMonday()); + const touched: Memory[] = []; + + for (const {subject, facts} of buckets) { + let node = mem.find(m => m.name === subject); + if (!node) { + node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []}; + mem.push(node); + } + this.appendFacts(node, facts); + const [e] = await this.llm.embedding(node.content); + if (e) node.embedding = e.embedding; + this.touch(node.name); + touched.push(node); } - if (updates.backlinks !== undefined) { - const backlinksList = updates.backlinks.length ? `[${updates.backlinks.map(l => `"${l}"`).join(', ')}]` : '[]'; - newFm = newFm.replace(/^backlinks:.*$/m, `backlinks: ${backlinksList}`); + if (touched.length) { + rebuildGraph(mem); + this.sync(memories); + (pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`; + for (const node of touched) this.reconcile(node, memories, options).catch(() => {}); + } else { + (pending as any).content = 'Nothing worth remembering.'; } - newFm = newFm.replace(/^modified:.*$/m, `modified: ${new Date().toISOString()}`); - - return `---\n${newFm}\n---\n\n${body}`; + return touched; } - private stripHeader(content: string): string { - return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart(); + /** Manual/cron entry point. scope 'touched' only reconciles docs with a pending Facts inbox. */ + async reconcileVault(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise { + const mem = this.unwrap(memories); + const targets = scope === 'all' ? mem : mem.filter(m => m.content.includes(FACTS_HEADING)); + await Promise.all(targets.map(node => this.reconcile(node, memories, options))); + this.sync(memories); } - private async docAgent(node: Memory, facts: string[], memories: Memory[], options: LLMRequest, tempName: string, week: {monday: string, sunday: string} | undefined, entry: {request: {abort?: () => void} | null}): Promise { - const {links: oldLinks} = extractMetadata(node.content); + /** + * Coalescing queue: if a doc is already reconciling, mark it dirty and abort the in-flight + * request. The loop below always re-reads node.content fresh, so nothing is ever dropped. + */ + private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise { + const key = node.name; + const existing = this.queues.get(key); + if (existing) { + existing.dirty = true; + existing.request?.abort?.(); + return existing.task; + } + + const entry = {dirty: false, request: null, task: Promise.resolve()}; + this.queues.set(key, entry); + const mem = this.unwrap(memories); + entry.task = (async () => { + do { + entry.dirty = false; + await this.reconcileDoc(node, mem, options, entry); + } while (entry.dirty); + })().finally(() => { + this.queues.delete(key); + rebuildGraph(mem); + this.sync(memories); + }); + return entry.task; + } + + private async reconcileDoc(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise { const currentBody = this.stripHeader(node.content); let update; try { - for(let i = 0; i < 3 && !update?.content; i++) { - const request = this.llm.ask(`New Facts:\n${facts.map(f => `- ${f}`).join('\n')}`, { + for (let i = 0; i < 2 && !update?.content; i++) { + const request = this.llm.ask(currentBody, { model: options.model, temperature: 0.3, schema: { description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true}, - content: {type: 'string', description: 'Rewritten document in markdown, without the frontmatter block', required: true}, + content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true}, }, - system: `You are a knowledge base editor. Rewrite the current document below so it incorporates the new facts. + system: `You are a knowledge base editor maintaining one document in an Obsidian-style vault. + +If the document has a "${FACTS_HEADING}" section, integrate every bullet under it into the appropriate part of the document, then remove the "${FACTS_HEADING}" section entirely. If there is no such section, just tidy the document per the rules below. + +Structure: follow this generic shape loosely, adapting section names/order to what the content actually needs (e.g. journal-style docs may want a timeline instead of "Details"): +\`\`\`markdown +${GENERIC_TEMPLATE} +\`\`\` Formatting rules: -- Use Obsidian-style markdown: # headings, **bold** to add emphasis, __italics__ for titles, terms, etc, bullet & numbered lists for grouped 1D data and tables for 2D data +- Use Obsidian-style markdown: # headings, **bold** for emphasis, bullet & numbered lists for grouped 1D data, tables for 2D data - Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]] -- Create links for specific entities (person, place, project, program) and abstract concepts (quantum mechanics, entropy) but skip generics (car, red, dog) +- Create links for specific entities (person, place, project, program) and abstract concepts, but skip generics (car, red, dog) - Keep the document concise, factual, and human-readable -- Resolve contradictions: the new facts always win — delete the outdated statement entirely, never keep both -- Later facts in the list override earlier ones +- Resolve contradictions: newer facts always win — delete the outdated statement entirely, never keep both - Do not add frontmatter blocks, filler, preamble, or AI commentary -${week ? '- This is a weekly journal entry.\n' : ''} -All nodes: -${this.listNodes(memories).map(n => n.name).join(', ') || 'none'} + +Other nodes in the vault (link to these instead of duplicating their content): +${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'} Current document: \`\`\`markdown ${currentBody} -\`\`\``} - ); +\`\`\``, + }); entry.request = request; update = await request; } } catch (err: any) { - if (err?.name === 'AbortError') return false; + if (err?.name === 'AbortError') return; throw err; } finally { entry.request = null; } - if(!update?.content) return false; - const newLinks = extractLinks(update.content).filter(l => l !== node.name && l !== tempName); - const newLinkSet = new Set(newLinks); - const oldLinkSet = new Set(oldLinks); - - for (const added of newLinkSet) { - if (!oldLinkSet.has(added)) { - const target = memories.find(m => m.name === added); - if (target) { - const {backlinks} = extractMetadata(target.content); - if (!backlinks.includes(node.name)) { - target.content = this.updateFrontmatter(target.content, { - backlinks: [...backlinks, node.name], - }); - } - } - } - } - for (const removed of oldLinkSet) { - if (!newLinkSet.has(removed)) { - const target = memories.find(m => m.name === removed); - if (target) { - const {backlinks} = extractMetadata(target.content); - target.content = this.updateFrontmatter(target.content, { - backlinks: backlinks.filter(b => b !== node.name), - }); - } - } - } - - const {backlinks} = extractMetadata(node.content); - node.description = node.name !== 'Person/User' ? update.description : 'All information about the current user'; - node.content = this.applyHeader(update.content, this.buildHeader(node, week, newLinks, backlinks)); + if (!update?.content) return; + node.description = node.name !== 'People/User' ? update.description : 'All information about the current user'; + node.content = this.touchHeader(node, update.content); const [e] = await this.llm.embedding(node.content); - if(e) node.embedding = e.embedding; - return true; + if (e) node.embedding = e.embedding; } private async factAgent(conversation: string, memories: Memory[], options: LLMRequest, weekKey: string): Promise { @@ -546,13 +473,13 @@ Rules: When extracting facts, you MUST also decide the exact destination path: - Use an existing node name if the facts clearly belong there -- All information primary about the user should go under "People/User" -- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide) +- All information primarily about the user should go under "People/User" +- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide) — you are not limited to any fixed list of collections, use whatever fits - For journal entries, use "Journal" Available nodes: - Journal -${this.listNodes(memories).filter(n => !n.name.includes('_temp_') && !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`, +${this.listNodes(memories).filter(n => !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`, tools: [{ name: 'facts_extract', description: 'Submit facts with their destination', diff --git a/tests/llm.spec.ts b/tests/llm.spec.ts new file mode 100644 index 0000000..2e887cd --- /dev/null +++ b/tests/llm.spec.ts @@ -0,0 +1,167 @@ + +import {describe, it, expect, vi, beforeEach} from 'vitest'; +import LLM from '../src/llm'; + +const {FakeProvider, providerLog} = vi.hoisted(() => { + const providerLog: any[] = []; + class FakeProvider { + model: string; + constructor(...args: any[]) { this.model = args[args.length - 1]; } + ask(message: string, opts: any) { + let aborted = false; + const p = (async () => { + const script = (globalThis as any).__scripts?.[this.model]; + const plan = script ? script(message, opts) : {text: ''}; + providerLog.push({model: this.model, message, system: opts.system, tools: (opts.tools || []).map((t: any) => t.name)}); + for (const c of plan.calls || []) { + if (aborted) break; + const tool = (opts.tools || []).find((t: any) => t.name === c.tool); + const id = c.id || `${c.tool}_${Math.random()}`; + const content = await tool.fn(c.args, opts.stream, null, id); + opts.history.push({role: 'tool', id, name: c.tool, args: c.args, content, timestamp: Date.now()}); + } + const text = plan.text ?? ''; + if (opts.stream && text) opts.stream({text, done: true}); + opts.history.push({role: 'assistant', content: text, timestamp: Date.now(), duration: 10, tps: 5}); + return text; + })(); + return Object.assign(p, {abort: () => { aborted = true; }}); + } + } + return {FakeProvider, providerLog}; +}); + +vi.mock('../src/antrhopic.ts', () => ({Anthropic: FakeProvider})); +vi.mock('../src/open-ai.ts', () => ({OpenAi: FakeProvider})); + +function makeAi(models: any) { + return {options: {llm: {models}}} as any; +} + +beforeEach(() => { + providerLog.length = 0; + (globalThis as any).__scripts = {}; +}); + +describe('LLM cross-provider interchangeability', () => { + it('runs identical tool calls the same way on an anthropic-backed model and an openai-backed model', async () => { + const ai = makeAi({ + claude: {proto: 'anthropic', token: 'x'}, + gpt: {proto: 'openai', token: 'y', host: 'http://local'}, + }); + const llm = new LLM(ai); + const calc = { + name: 'calc_add', + description: 'Add two numbers', + args: {a: {type: 'number', required: true}, b: {type: 'number', required: true}}, + fn: (args: any) => String(args.a + args.b), + }; + + (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'}); + (globalThis as any).__scripts.gpt = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'}); + + const historyA: any[] = [], historyB: any[] = []; + const respA = await llm.ask('add 2 and 3', {model: 'claude', tools: [calc], history: historyA}); + const respB = await llm.ask('add 2 and 3', {model: 'gpt', tools: [calc], history: historyB}); + + expect(respA).toBe('Result: 5'); + expect(respB).toBe('Result: 5'); + expect(providerLog.find(l => l.model === 'claude')!.tools).toContain('calc_add'); + expect(providerLog.find(l => l.model === 'gpt')!.tools).toContain('calc_add'); + + // tool timing gets recomputed from real execution regardless of proto + for (const h of [historyA.find(h => h.name === 'calc_add'), historyB.find(h => h.name === 'calc_add')]) { + expect(h.content).toBe('5'); + expect(typeof h.duration).toBe('number'); + expect(typeof h.tps).toBe('number'); + } + }); + + it('lets the same shared history flow across model + proto swaps with different system prompts', async () => { + const ai = makeAi({ + claude: {proto: 'anthropic', token: 'x'}, + gpt: {proto: 'openai', token: 'y', host: 'http://local'}, + }); + const llm = new LLM(ai); + const history: any[] = []; + + (globalThis as any).__scripts.claude = () => ({text: 'Hi from claude'}); + (globalThis as any).__scripts.gpt = () => ({text: 'Hi from gpt'}); + + const r1 = await llm.ask('hello', {model: 'claude', system: 'You are terse.', history}); + const r2 = await llm.ask('follow up', {model: 'gpt', system: 'You are verbose.', history}); + + expect(r1).toBe('Hi from claude'); + expect(r2).toBe('Hi from gpt'); + expect(history.filter(h => h.role === 'assistant').map(h => h.content)).toEqual(['Hi from claude', 'Hi from gpt']); + expect(providerLog[0].system).toContain('You are terse.'); + expect(providerLog[1].system).toContain('You are verbose.'); + }); + + it('exposes MCP tools the same way no matter which proto backs the model', async () => { + const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); + const llm = new LLM(ai); + const mcp = [{name: 'weather', host: 'http://mcp.local'}]; + + global.fetch = vi.fn(async (url: string, opts?: any) => { + if (url.endsWith('/tools')) { + return {json: async () => ({tools: [{name: 'lookup', description: 'Look up weather', inputSchema: {properties: {city: {type: 'string'}}, required: ['city']}}]})} as any; + } + const body = JSON.parse(opts.body); + return {json: async () => ({content: [{text: `Sunny in ${body.arguments.city}`}]})} as any; + }) as any; + + for (const model of ['claude', 'gpt']) { + (globalThis as any).__scripts[model] = () => ({calls: [{tool: 'weather_lookup', args: {city: 'Rome'}}], text: 'done'}); + const history: any[] = []; + await llm.ask('weather?', {model, mcp, history}); + expect(history.find(h => h.name === 'weather_lookup')?.content).toBe('Sunny in Rome'); + } + }); + + it('exposes and resolves skill documents identically across protos', async () => { + const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); + const llm = new LLM(ai); + const skills = [{name: 'Onboarding', description: 'How to onboard a user', content: 'Step 1...'}]; + + for (const model of ['claude', 'gpt']) { + (globalThis as any).__scripts[model] = () => ({calls: [{tool: 'skill_read', args: {name: 'Onboarding'}}], text: 'done'}); + const history: any[] = []; + await llm.ask('onboard me', {model, skills, history}); + expect(history.find(h => h.name === 'skill_read')?.content).toContain('Step 1...'); + } + }); + + it('delegate agent mutates the shared history directly and backfills the orchestrator response, across protos', async () => { + const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); + const llm = new LLM(ai); + const history: any[] = [{role: 'user', content: 'research quantum computing'}]; + const researcher = {name: 'researcher', system: 'You research topics.', delegate: true, model: 'gpt'}; + + (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'agent_researcher', args: {}}], text: ''}); + (globalThis as any).__scripts.gpt = () => ({text: 'Quantum computers use qubits.'}); + + const resp = await llm.ask('go', {model: 'claude', agents: [researcher], history}); + + expect(resp).toBe('Quantum computers use qubits.'); + expect(history.some(h => h.role === 'assistant' && h.content === 'Quantum computers use qubits.')).toBe(true); + expect(history.find(h => h.name === 'agent_researcher')?.content).toBe(''); + }); + + it('regular (non-delegate) subagent keeps its own isolated history separate from the parent, across protos', async () => { + const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); + const llm = new LLM(ai); + const history: any[] = []; + const summarizer = {name: 'summarizer', system: 'You summarize text.', model: 'gpt'}; + + (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'subagent_summarizer', args: {context: 'a long article', instructions: 'summarize it'}}], text: 'Summary: short version'}); + (globalThis as any).__scripts.gpt = () => ({text: 'short version'}); + + const resp = await llm.ask('summarize this', {model: 'claude', agents: [summarizer], history}); + + expect(resp).toBe('Summary: short version'); + expect(history.find(h => h.name === 'subagent_summarizer')?.content).toBe('short version'); + // isolated history - subagent's own assistant turn never leaks into the parent + expect(history.some(h => h.role === 'assistant' && h.content === 'short version')).toBe(false); + }); +}); diff --git a/tests/memory.spec.ts b/tests/memory.spec.ts new file mode 100644 index 0000000..6e2da0c --- /dev/null +++ b/tests/memory.spec.ts @@ -0,0 +1,256 @@ +import {describe, it, expect, vi, beforeEach} from 'vitest'; +import {MemoryManager, MemoryCache, rebuildGraph, Memory} from '../src/memory'; + +function makeMemory(overrides: Partial = {}): Memory { + return { + name: 'Test/Doc', + description: '', + content: '', + embedding: [], + links: [], + backlinks: [], + ...overrides, + }; +} + +function makeLLM() { + return { + embedding: vi.fn(async (_text: string) => [{embedding: [1, 0, 0]}]), + ask: vi.fn(async () => undefined), + }; +} + +describe('rebuildGraph', () => { + it('extracts [[WikiLinks]] from content, excluding self-links', () => { + const a = makeMemory({name: 'A', content: '[[B]] and [[A]] and [[C]]'}); + const b = makeMemory({name: 'B', content: 'no links here'}); + const mem = [a, b]; + + rebuildGraph(mem); + + expect(a.links).toEqual(['B', 'C']); + expect(b.links).toEqual([]); + }); + + it('computes backlinks only for links that resolve to a real node', () => { + const a = makeMemory({name: 'A', content: '[[B]] [[Missing]]'}); + const b = makeMemory({name: 'B', content: ''}); + const mem = [a, b]; + + rebuildGraph(mem); + + expect(b.backlinks).toEqual(['A']); + expect(mem.find(m => m.name === 'Missing')).toBeUndefined(); + }); + + it('resets stale backlinks on every rebuild (no leftover from a removed link)', () => { + const a = makeMemory({name: 'A', content: '[[B]]'}); + const b = makeMemory({name: 'B', content: ''}); + const mem = [a, b]; + rebuildGraph(mem); + expect(b.backlinks).toEqual(['A']); + + a.content = 'no more links'; + rebuildGraph(mem); + expect(b.backlinks).toEqual([]); + }); +}); + +describe('MemoryCache', () => { + it('finds nearest neighbor by embedding via KD-tree search', () => { + const close = makeMemory({name: 'Close', embedding: [1, 0, 0]}); + const far = makeMemory({name: 'Far', embedding: [0, 0, 1]}); + const cache = new MemoryCache([close, far]); + + const results = cache.search([1, 0, 0], 1); + + expect(results[0].name).toBe('Close'); + }); + + it('rebuilds the tree on add/update/remove', () => { + const cache = new MemoryCache([makeMemory({name: 'A', embedding: [1, 0, 0]})]); + cache.add(makeMemory({name: 'B', embedding: [0, 1, 0]})); + expect(cache.search([0, 1, 0], 1)[0].name).toBe('B'); + + cache.remove('B'); + expect(cache.search([0, 1, 0], 1)[0]?.name).not.toBe('B'); + }); +}); + +describe('MemoryManager.forget', () => { + it('removes the node and recomputes backlinks for the rest of the graph', () => { + const llm = makeLLM(); + const mgr = new MemoryManager(llm); + const a = makeMemory({name: 'A', content: '[[B]]'}); + const b = makeMemory({name: 'B', content: '[[C]]'}); + const c = makeMemory({name: 'C', content: ''}); + const mem = [a, b, c]; + rebuildGraph(mem); + expect(c.backlinks).toEqual(['B']); + + const ok = mgr.forget('B', mem); + + expect(ok).toBe(true); + expect(mem.find(m => m.name === 'B')).toBeUndefined(); + expect(a.links).toEqual(['B']); + expect(c.backlinks).toEqual([]); + }); + + it('returns false for an unknown name', () => { + const mgr = new MemoryManager(makeLLM()); + expect(mgr.forget('Nope', [makeMemory({name: 'A'})])).toBe(false); + }); +}); + +describe('MemoryManager.recollect', () => { + it('orders vector matches first, then expands one hop via links', async () => { + const llm = makeLLM(); + llm.embedding.mockResolvedValue([{embedding: [1, 0, 0]}]); + const mgr = new MemoryManager(llm); + + const near = makeMemory({name: 'Near', embedding: [1, 0, 0], content: '[[Linked]]'}); + const linked = makeMemory({name: 'Linked', embedding: [0, 0, 1], content: ''}); + const far = makeMemory({name: 'Far', embedding: [0, 1, 0], content: ''}); + const mem = [near, linked, far]; + rebuildGraph(mem); + + const result = await mgr.recollect('query', mem, 1, 1); + + expect(result.map(r => r.name)).toEqual(['Near', 'Linked']); + }); + + it('returns [] when there are no memories', async () => { + const mgr = new MemoryManager(makeLLM()); + expect(await mgr.recollect('q', [])).toEqual([]); + }); +}); + +describe('MemoryManager.memorize (fast path)', () => { + let llm: ReturnType; + let mgr: MemoryManager; + + beforeEach(() => { + llm = makeLLM(); + mgr = new MemoryManager(llm); + }); + + it('pushes a pending tool message, then resolves it to links once facts land', async () => { + llm.ask.mockImplementation(async (_prompt: string, opts: any) => { + if (opts.tools) { + opts.tools[0].fn({destination: 'Projects/Oxide', facts: 'Uses a hybrid memory system'}); + return undefined; + } + return {description: 'd', content: '# doc'}; + }); + + const history: any[] = [{role: 'user', content: 'we use a hybrid memory system'}]; + const touched = await mgr.memorize(history, [], {model: 'test'} as any); + + const pending = history.find(h => h.name === 'memory_process'); + expect(pending).toBeDefined(); + expect(pending.content).toContain('[[Projects/Oxide]]'); + expect(touched.map(t => t.name)).toEqual(['Projects/Oxide']); + }); + + it('creates a new node and appends facts under "## Facts" without calling the doc LLM', async () => { + llm.ask.mockImplementation(async (_prompt: string, opts: any) => { + if (opts.tools) opts.tools[0].fn({destination: 'People/Sarah', facts: 'Works at Acme, Likes hiking'}); + return undefined; + }); + + const mem: Memory[] = []; + await mgr.memorize([{role: 'user', content: 'Sarah works at Acme and likes hiking'}] as any, mem, {model: 'test'} as any); + + const node = mem.find(m => m.name === 'People/Sarah')!; + expect(node).toBeDefined(); + expect(node.content).toContain('## Facts'); + expect(node.content).toContain('- Works at Acme'); + expect(node.content).toContain('- Likes hiking'); + // doc reconciler LLM (schema call) should NOT have been awaited synchronously in this fast path assertion + }); + + it('routes "journal" destination to Journal/{weekMonday}', async () => { + llm.ask.mockImplementation(async (_prompt: string, opts: any) => { + if (opts.tools) opts.tools[0].fn({destination: 'journal', facts: 'Shipped v1'}); + return undefined; + }); + + const mem: Memory[] = []; + const touched = await mgr.memorize([{role: 'user', content: 'shipped v1 today'}] as any, mem, {model: 'test'} as any); + + expect(touched[0].name).toMatch(/^Journal\/\d{4}-\d{2}-\d{2}$/); + }); + + it('reports nothing to remember when no facts are extracted', async () => { + llm.ask.mockResolvedValue(undefined); // tools present but fn never called + + const history: any[] = [{role: 'user', content: 'hey'}]; + const touched = await mgr.memorize(history, [], {model: 'test'} as any); + + expect(touched).toEqual([]); + expect(history.find(h => h.name === 'memory_process').content).toBe('Nothing worth remembering.'); + }); + + it('returns [] and does nothing for an empty conversation', async () => { + const touched = await mgr.memorize([], [], {model: 'test'} as any); + expect(touched).toEqual([]); + expect(llm.ask).not.toHaveBeenCalled(); + }); +}); + +describe('MemoryManager reconcileVault', () => { + it('integrates the "## Facts" section via the doc LLM and removes it', async () => { + const llm = makeLLM(); + llm.ask.mockResolvedValue({description: 'Tidy summary', content: '# Doc\n\nIntegrated fact.'}); + const mgr = new MemoryManager(llm); + + const node = makeMemory({ + name: 'Projects/Oxide', + content: '---\nname: Projects/Oxide\n---\n\n# Doc\n\n## Facts\n- some raw fact\n', + }); + const mem = [node]; + + await mgr.reconcileVault(mem, {model: 'test'} as any, 'all'); + + expect(node.content).not.toContain('## Facts'); + expect(node.content).toContain('Integrated fact.'); + expect(node.description).toBe('Tidy summary'); + }); + + it('only targets docs with a pending Facts inbox when scope is "touched"', async () => { + const llm = makeLLM(); + llm.ask.mockResolvedValue({description: 'd', content: '# clean'}); + const mgr = new MemoryManager(llm); + + const dirty = makeMemory({name: 'A', content: '## Facts\n- x'}); + const clean = makeMemory({name: 'B', content: '# already tidy'}); + await mgr.reconcileVault([dirty, clean], {model: 'test'} as any, 'touched'); + + expect(dirty.content).toContain('# clean'); // rewritten (frontmatter now wraps it) + expect(clean.content).toBe('# already tidy'); // untouched, never queued + }); +}); + +describe('MemoryManager reconcile coalescing', () => { + it('coalesces a second call while one is in-flight: marks dirty, aborts, reuses the same task promise', () => { + const llm = makeLLM(); + const abort = vi.fn(); + let calls = 0; + llm.ask.mockImplementation(() => { + calls++; + const pending: any = new Promise(() => {}); // never resolves in this test + pending.abort = abort; + return pending; + }); + const mgr: any = new MemoryManager(llm); + const node = makeMemory({name: 'Q', content: '# Q\n\n## Facts\n- f'}); + const mem = [node]; + + const p1 = mgr.reconcile(node, mem, {model: 'test'}); + const p2 = mgr.reconcile(node, mem, {model: 'test'}); + + expect(p2).toBe(p1); // same in-flight task, not a new queue entry + expect(abort).toHaveBeenCalledTimes(1); // second call aborted the in-flight request + expect(calls).toBe(1); // no second ask() fired synchronously — it'll rerun via the dirty loop + }); +});