diff --git a/README.md b/README.md index 330d7fc..d92b849 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ console.log(chunks); // Manually compile history into memories at end of conversation // Happens automatically when coverstaions are compressed -await ai.language.updateMemory(history, memory); +await ai.language.memorize(history, memory); // Summarize text const summary = await ai.language.summarize(longText, 200); diff --git a/package.json b/package.json index 6d39719..201b0b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/ai-utils", - "version": "1.4.4", + "version": "1.4.5", "description": "AI Utility library", "author": "Zak Timson", "license": "MIT", diff --git a/src/helpers.ts b/src/helpers.ts index 29d128d..b38992d 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -7,10 +7,24 @@ export type MemoryNode = { backlinks: string[]; } -export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[] { +export function extractLinks(content: string): string[] { + if (!content) return []; + const matches = content.matchAll(/\[\[([^\]|]+)(?:\|[^\]]*)?\]\]/g); + return [...new Set([...matches].map(m => m[1].trim()))]; +} + +export function rebuildGraph(memories: Memory[] | MemoryCache): MemoryNode[] { const mems = memories instanceof MemoryCache ? memories.memories : memories; const nameSet = new Set(mems.map(m => m.name)); - const ghosts = new Set(); + + for (const m of mems) m.links = extractLinks(m.content).filter(l => l !== m.name); + for (const m of mems) m.backlinks = []; + for (const m of mems) { + for (const link of m.links) { + const target = mems.find(t => t.name === link); + if (target) target.backlinks.push(m.name); + } + } const nodes: MemoryNode[] = mems.map(m => ({ name: m.name, @@ -19,6 +33,7 @@ export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[] backlinks: m.backlinks, })); + const ghosts = new Set(); for (const node of nodes) { for (const link of node.links) { if (!nameSet.has(link)) ghosts.add(link); @@ -31,30 +46,28 @@ export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[] name, missing: true, links: [], - backlinks: nodes - .filter(n => n.links.includes(name)) - .map(n => n.name), - })) + backlinks: nodes.filter(n => n.links.includes(name)).map(n => n.name), + })), ]; } -export function renderMemoryGraph(nodes) { +export function renderMemoryGraph(nodes: MemoryNode[]): string { if (!nodes.length) return 'No memories yet.'; - const groups = new Map(); + const groups = new Map(); for (const node of nodes) { const [prefix, ...rest] = node.name.split('/'); const group = rest.length ? prefix : 'Root'; const label = rest.length ? rest.join('/') : node.name; if (!groups.has(group)) groups.set(group, []); - groups.get(group).push({...node, label}); + groups.get(group)!.push({...node, label}); } const ghostCount = nodes.filter(n => n.missing).length; const lines = [`Memory Graph (${nodes.length} nodes, ${ghostCount} ghost${ghostCount === 1 ? '' : 's'})`, '']; for (const group of [...groups.keys()].sort()) { - const items = groups.get(group).sort((a, b) => a.label.localeCompare(b.label)); + const items = groups.get(group)!.sort((a, b) => a.label.localeCompare(b.label)); lines.push(`${group}/`); items.forEach((n, i) => { const last = i === items.length - 1; diff --git a/src/kd-tree.ts b/src/kd-tree.ts index db32ec2..dd43378 100644 --- a/src/kd-tree.ts +++ b/src/kd-tree.ts @@ -103,9 +103,10 @@ class BoundedMaxHeap { export class KDTree { private root: KDNode | null = null; private _size = 0; - private readonly dims: number; private readonly distanceFn: (a: number[], b: number[]) => number; + readonly dims: number; + /** * @param dims Dimensionality of all vectors (must be consistent). * @param metric Distance metric to use. Default: "euclidean". diff --git a/src/llm.ts b/src/llm.ts index 1ed9fd8..8459eb9 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -1,4 +1,4 @@ -import {snakeCase} from '@ztimson/utils'; +import {clean, snakeCase} from '@ztimson/utils'; import {AbortablePromise, Ai} from './ai.ts'; import {Anthropic} from './antrhopic.ts'; import {OpenAi} from './open-ai.ts'; @@ -132,7 +132,7 @@ class LLM { return { name: toolName, description: `${a.delegate ? 'Delegate to ' : ''}Subagent: ${a.description || a.name}`, - args: ({ + args: clean({ context: !a.delegate ? {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true} : undefined, instructions: {type: 'string', description: 'Detailed instructions for subagent to complete', required: true}, }), @@ -143,8 +143,6 @@ class LLM { .map(name => allAgents.find(x => x.name === name)) .filter((x): x is Agent => !!x && x.name !== a.name); - // Delegate continues the SAME live conversation - no new user turn needed, - // `history` is always current (shared, mutated in place) by the time this runs const q = a.delegate ? '' : `${args.instructions}${args.context ? `\n\n${args.context}` : ''}`; const request = this.ask(q, { @@ -218,7 +216,7 @@ ${a.system}`, if(!skills?.length) return {prompt: '', tools: []}; const list = skills.map(s => `- ${s.name}: ${s.description}`).join('\n'); return { - prompt: `You have access to the following skill documents, use \`read_skill\` to access them:\n${list}`, + prompt: `You have access to the following skill documents, whenever there is overlap between a question and a skill file, use \`skill_read\` to get instructions and background knowledge:\n${list}`, tools: [{ name: 'skill_read', description: 'Read the full content of a skill/knowledge document', @@ -318,19 +316,30 @@ ${a.system}`, } else listed.push(r); } - prompts.unshift(`You have access to the following memory files: -${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')} + prompts.unshift(`You have a background memory process which has prefetched relevant information${mem.update ? ' and will create new memories from this conversation' : ''} for you +Assume it is perfect and never mention this process to anyone ever +Always use your memories to craft a personalized response, they contain links / [[wiki links]] which you use navigate between them +${mem.tool ? `You can access memory files via the \`memory_search\` and \`memory_recall\` tools +When you need information about the user, \`memory_recall\` \`People/User\` before asking (fetch if not included bellow) +When you need information not provided, attempt 1-3 \`memory_search\` calls with unique queries before asking` : ''} + ${preloaded.length ? ` -Relevant memories have been preloaded: -${preloaded.map(r => ` -**${r.name}** -${r.description} +Prefetched Memories (Most relevant first): + +${preloaded.map(r => `Memory: ${r.name} +Description: ${r.description} +Linked: ${[r.links, ...r.backlinks].join(', ')} +\`\`\` ${r.content} -`).join('\n---\n')} -` : ''}${listed.length ? ` -Additional relevant memories (use \`memory_recall\`): -${listed.map(r => r.name).join(', ')} -` : ''}`.trim()); +\`\`\``).join('\n\n')}` : ''} + +${mem.tool && listed.length ? listed.map(r => `Memory: ${r.name} +Description: ${r.description} +Linked: ${[r.links, ...r.backlinks].join(', ')} +`).join('\n\n') : ''} + +${mem.tool ? `Full memory list: +${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')}` : ''}`.trim()) } if(mem.tool) tools.push(this.memoryManager.tools.read(mem.memory)); } @@ -374,14 +383,6 @@ ${listed.map(r => r.name).join(', ')} return Object.assign(promise, {abort}); } - /** - * Digest full conversation history into memory documents. - * Call on session end to persist the conversation. - */ - async updateMemory(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest = {}): Promise { - return this.memoryManager.memorize(history, memories, {model: this.defaultModel, ...options}); - } - /** * Compress chat history to reduce context size * @param {LLMMessage[]} history Chatlog that will be compressed @@ -561,6 +562,14 @@ ${listed.map(r => r.name).join(', ')} }; } + /** + * Digest full conversation history into memory documents. + * Call on session end to persist the conversation. + */ + async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest = {}): Promise { + return this.memoryManager.memorize(history, memories, {model: this.defaultModel, ...options}); + } + /** * Create a summary of some text * @param {string} text Text to summarize diff --git a/src/memory.ts b/src/memory.ts index bc146c0..96c3d53 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -1,3 +1,4 @@ +import {MemoryNode, rebuildGraph} from './helpers.ts'; import {LLMRequest, LLMMessage} from './llm.ts'; import {AiTool} from './tools.ts'; import {KDPoint, KDTree} from './kd-tree.ts'; @@ -12,79 +13,6 @@ const GENERIC_TEMPLATE = `# {{Title}} ## Related`; -export class MemoryCache { - private tree: KDTree; - public memories: Memory[]; - - get length() { return this.memories.length; } - - constructor(memories: Memory[]) { - this.memories = memories; - this.tree = this.buildTree(); - } - - private buildTree(): KDTree { - const embedded = this.memories.filter(m => m.embedding?.length); - if (!embedded.length) return new KDTree(0); - - const dims = embedded[0].embedding.length; - const points: KDPoint[] = embedded.map(m => ({ - vector: m.embedding, - payload: {name: m.name, description: m.description}, - })); - - return new KDTree(dims, 'cosine', points); - } - - search(query: number[], limit: number): MemoryRef[] { - const results = this.tree.knn(query, limit); - return results.map(r => r.point.payload); - } - - add(memory: Memory): void { - this.memories.push(memory); - rebuildGraph(this.memories); - this.rebuild(); - } - - update(memory: Memory): void { - const idx = this.memories.findIndex(m => m.name === memory.name); - if (idx !== -1) { - this.memories[idx] = memory; - } else { - this.memories.push(memory); - } - rebuildGraph(this.memories); - this.rebuild(); - } - - remove(name: string): void { - const idx = this.memories.findIndex(m => m.name === name); - if (idx !== -1) { - this.memories.splice(idx, 1); - rebuildGraph(this.memories); - this.rebuild(); - } - } - - rebuild(): void { - this.tree = this.buildTree(); - } -} - -export type MemoryOptions = { - /** Memory object */ - memory: Memory[] | MemoryCache; - /** Inject N memories into the system prompt */ - inject?: boolean; - /** expose recall tool to LLM */ - tool?: boolean; - /** Update memory on compression */ - update?: boolean; - /** Max context size of memories to inject to each call (removed immediately after use) */ - maxTokens?: number; -} - export type Memory = { name: string; description: string; @@ -104,23 +32,6 @@ type FactBucket = { facts: string[]; } -function extractLinks(content: string): string[] { - if (!content) return []; - const matches = content.matchAll(/\[\[([^\]]+)\]\]/g); - return [...new Set([...matches].map(m => m[1].trim()))]; -} - -export function rebuildGraph(memories: Memory[]): void { - for (const m of memories) m.links = extractLinks(m.content).filter(l => l !== m.name); - for (const m of memories) m.backlinks = []; - for (const m of memories) { - for (const link of m.links) { - const target = memories.find(t => t.name === link); - if (target) target.backlinks.push(m.name); - } - } -} - function dedupeFacts(facts: string[]): string[] { const seen = new Map(); for (const f of facts) { @@ -141,12 +52,132 @@ function cosineDistance(a: number[], b: number[]): number { return denom === 0 ? 1 : 1 - dot / denom; } -function getWeekMonday(date: Date = new Date()): string { - const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate())); - const day = d.getUTCDay(); - const diff = day === 0 ? -6 : 1 - day; - d.setUTCDate(d.getUTCDate() + diff); - return d.toISOString().slice(0, 10); +function cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] { + return memories + .filter(m => m.embedding?.length) + .map(m => ({ref: {name: m.name, description: m.description}, distance: cosineDistance(query, m.embedding)})) + .sort((a, b) => a.distance - b.distance) + .slice(0, limit) + .map(s => s.ref); +} + +export class MemoryCache { + private tree!: KDTree; + public memories: Memory[]; + public nodes: MemoryNode[] = []; + + get length() { return this.memories.length; } + + constructor(memories: Memory[]) { + this.memories = memories; + this.rebuild(); + } + + private buildTree(): KDTree { + const embedded = this.memories.filter(m => m.embedding?.length); + if (!embedded.length) return new KDTree(0); + + const dims = embedded[0].embedding.length; + const points: KDPoint[] = embedded.map(m => ({ + vector: m.embedding, + payload: {name: m.name, description: m.description}, + })); + + return new KDTree(dims, 'cosine', points); + } + + search(query: number[], limit: number): MemoryRef[] { + if (!this.tree || this.tree.dims === 0) return []; + return this.tree.knn(query, limit).map(r => r.point.payload); + } + + add(memory: Memory): void { + this.memories.push(memory); + this.rebuild(); + } + + update(memory: Memory): void { + const idx = this.memories.findIndex(m => m.name === memory.name); + if (idx !== -1) this.memories[idx] = memory; + else this.memories.push(memory); + this.rebuild(); + } + + remove(name: string): void { + const idx = this.memories.findIndex(m => m.name === name); + if (idx !== -1) { + this.memories.splice(idx, 1); + this.rebuild(); + } + } + + rebuild(): void { + this.nodes = rebuildGraph(this.memories); + this.tree = this.buildTree(); + } +} + +class MemoryAccessor { + readonly list: Memory[]; + private readonly cache: MemoryCache | null; + + constructor(memories: Memory[] | MemoryCache) { + this.cache = memories instanceof MemoryCache ? memories : null; + this.list = this.cache ? this.cache.memories : memories; + } + + find(name: string): Memory | undefined { + return this.list.find(m => m.name === name); + } + + commit(): MemoryNode[] { + if (this.cache) { + this.cache.rebuild(); + return this.cache.nodes; + } + return rebuildGraph(this.list); + } + + ghosts(): string[] { + const nodes = this.cache ? this.cache.nodes : rebuildGraph(this.list); + return nodes.filter(n => n.missing).map(n => n.name); + } + + search(vector: number[], limit: number): MemoryRef[] { + return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit); + } + + forget(name: string): boolean { + const idx = this.list.findIndex(m => m.name === name); + if (idx === -1) return false; + this.list.splice(idx, 1); + this.commit(); + return true; + } + + async backfillEmbeddings(llm: any): Promise { + const missing = this.list.filter(m => !m.embedding?.length); + if (!missing.length) return 0; + await Promise.all(missing.map(async node => { + const [e] = await llm.embedding(node.content); + if (e) node.embedding = e.embedding; + })); + this.commit(); + return missing.length; + } +} + +export type MemoryOptions = { + /** Memory object */ + memory: Memory[] | MemoryCache; + /** Inject N memories into the system prompt */ + inject?: boolean; + /** expose recall tool to LLM */ + tool?: boolean; + /** Update memory on compression */ + update?: boolean; + /** Max context size of memories to inject to each call (removed immediately after use) */ + maxTokens?: number; } export class MemoryManager { @@ -159,21 +190,6 @@ export class MemoryManager { }>(); tools = { - read: (memories: Memory[] | MemoryCache): AiTool => ({ - name: 'memory_recall', - description: 'Read the full content of a memory document', - args: { - name: {type: 'string', description: 'Exact memory name', required: true}, - }, - fn: (args: any) => { - const mems = this.unwrap(memories); - const mem = mems.find(m => m.name === args.name); - if (!mem) return 'Document not found'; - this.touch(mem.name); - return mem.content; - }, - }), - forget: (memories: Memory[] | MemoryCache): AiTool => ({ name: 'memory_forget', description: 'Permanently delete a memory document and clean up all references to it', @@ -185,68 +201,50 @@ export class MemoryManager { return result ? `Forgotten: ${args.name}` : `Not found: ${args.name}`; }, }), + + read: (memories: Memory[] | MemoryCache): AiTool => ({ + name: 'memory_recall', + description: 'Read the full content of a memory document', + args: { + name: {type: 'string', description: 'Exact memory name', required: true}, + }, + fn: (args: any) => { + const mem = this.access(memories).find(args.name); + if (!mem) return 'Document not found'; + this.touch(mem.name); + return mem.content; + }, + }), + + search: (memories: Memory[] | MemoryCache): AiTool => ({ + name: 'memory_search', + description: 'Use embeddings to find the MOST relevant memories, even if NOT relevant', + args: { + query: {type: 'string', description: 'What to look for in the memories', required: true}, + limit: {type: 'number', description: 'Number of memories to return', default: 1}, + }, + fn: async ({query, limit}) => { + const mem = await this.recollect(query, memories, limit) + return mem.map(m => `Memory: ${m.name} +Description: ${m.description} +Links: ${[...m.links, ...m.backlinks].join(', ')} +\`\`\` +${m.content} +\`\`\``).join('\n\n'); + }, + }), }; constructor(private llm: any) {} - private ghostNodes(memories: Memory[]): string[] { - const names = new Set(memories.map(m => m.name)); - const ghosts = new Set(); - for (const m of memories) { - for (const link of m.links) { - if (!names.has(link)) ghosts.add(link); - } - } - return [...ghosts]; - } - static normalize(m?: Memory[] | MemoryCache | MemoryOptions) { - if(!m) return null; + if (!m) return null; const raw = m instanceof MemoryCache || Array.isArray(m); return raw ? {memory: m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m}; } - private unwrap(memories: Memory[] | MemoryCache): Memory[] { - return memories instanceof MemoryCache ? memories.memories : memories; - } - - private sync(memories: Memory[] | MemoryCache): void { - if (memories instanceof MemoryCache) memories.rebuild(); - } - - private parseFrontmatter(content: string): {fm: Map, body: string} { - const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); - if (!match) return {fm: new Map(), body: content}; - const fm = new Map(); - for (const line of match[1].split('\n')) { - const i = line.indexOf(':'); - if (i === -1) continue; - fm.set(line.slice(0, i).trim(), line.slice(i + 1).trim()); - } - return {fm, body: match[2]}; - } - - private writeFrontmatter(fm: Map, body: string): string { - const lines = [...fm.entries()].map(([k, v]) => `${k}: ${v}`); - return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`; - } - - private stripHeader(content: string): string { - return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart(); - } - - private touchHeader(node: Memory, body: string): string { - const {fm} = this.parseFrontmatter(node.content); - fm.set('name', node.name); - fm.set('description', node.description || ''); - fm.set('modified', new Date().toISOString()); - return this.writeFrontmatter(fm, body); - } - - private ensureDoc(node: Memory): void { - if (node.content) return; - const title = node.name.split('/').pop() ?? node.name; - node.content = this.touchHeader(node, `# ${title}\n`); + private access(memories: Memory[] | MemoryCache): MemoryAccessor { + return new MemoryAccessor(memories); } private appendFacts(node: Memory, facts: string[]): void { @@ -260,137 +258,78 @@ export class MemoryManager { node.content = this.touchHeader(node, newBody); } - decay() { - for(const [name, ttl] of this.recentlyTouched) { - if(ttl <= 1) this.recentlyTouched.delete(name); - else this.recentlyTouched.set(name, ttl - 1); - } + private ensureDoc(node: Memory): void { + if (node.content) return; + const title = node.name.split('/').pop() ?? node.name; + node.content = this.touchHeader(node, `# ${title}\n`); } - touch(name: string, ttl = 2) { - this.recentlyTouched.set(name, ttl); - } + private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest, weekKey: string): Promise { + const ghosts = store.ghosts(); - getTouched(): string[] { - return [...this.recentlyTouched.keys()]; - } + const response = await this.llm.ask(conversation, { + model: options.model, + temperature: 0.2, + system: `You are a fact extractor to build obsidian knowledge vaults. +Analyze this conversation and extract facts worth remembering long-term. - forget(name: string, memories: Memory[] | MemoryCache): boolean { - const mem = this.unwrap(memories); - const idx = mem.findIndex(m => m.name === name); - if (idx === -1) return false; +Rules: +- Always extract facts that the user explicitly told you to remember +- ONLY extract current facts the USER explicitly stated about themselves, their work, projects or decisions that were MADE during this conversation +- DO NOT extract greetings, pleasantries, or generic exchanges +- DO NOT extract deltas or changes in facts; ONLY the end fact +- DO NOT extract anything the AI/assistant itself said +- If nothing worth remembering was said, return an empty buckets array - mem.splice(idx, 1); - rebuildGraph(mem); - this.sync(memories); - return true; - } +When extracting facts, you MUST also decide the exact destination path: +- Reuse node names (including ghost) as much as possible IF the facts belongs there +- All information primarily about the user should go under "People/User" +- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide) — you are not limited to any fixed list of collections, use whatever fits +- For journal entries, use "Journal" - async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise { - const mem = this.unwrap(memories); - if (!mem.length) return []; +Available nodes: +- Journal +${this.listNodes(store.list).filter(n => !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'} +${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, + schema: { + buckets: {type: 'array', description: 'Groups of facts to remember, each assigned to a different node. Return an empty array if there is nothing worth storing in an obsidian vault', items: { + type: 'object', items: { + subject: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide"), or "Journal"', required: true}, + facts: { + type: 'array', + description: 'Facts to store at this destination', + items: {type: 'string', description: 'A single fact'}, + }, + }, + }, + }, + }, + }); - const [e] = await this.llm.embedding(query); - if (!e) return []; - - let vectorResults: MemoryRef[]; - if (memories instanceof MemoryCache) vectorResults = memories.search(e.embedding, limit); - else vectorResults = this.cosineSearch(e.embedding, mem, limit); - const found = new Set(vectorResults.map(r => r.name)); - - if (graphDepth > 0) { - const frontier = [...found]; - for (let depth = 0; depth < graphDepth; depth++) { - const next: string[] = []; - for (const name of frontier) { - const node = mem.find(m => m.name === name); - if (!node) continue; - for (const link of node.links) { - if (!found.has(link) && mem.find(m => m.name === link)) { - found.add(link); - next.push(link); - } - } - } - frontier.splice(0, frontier.length, ...next); - if (!frontier.length) break; - } + const buckets = new Map(); + for(const bucket of response.buckets ?? []) { + const subject = bucket.subject.trim().toLowerCase() === 'journal' + ? `Journal/${weekKey}` : bucket.subject.trim(); + const facts = buckets.get(subject) ?? []; + facts.push(...dedupeFacts(bucket.facts)); + buckets.set(subject, facts); } - const vectorOrder = vectorResults.map(r => r.name); - const graphExpansions = [...found].filter(n => !vectorOrder.includes(n)); - const ordered = [...vectorOrder, ...graphExpansions]; - return ordered.map(n => mem.find(m => m.name === n)!).filter(Boolean); + return buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})); } - private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] { - const scored = memories - .filter(m => m.embedding?.length) - .map(m => ({ - ref: {name: m.name, description: m.description}, - distance: cosineDistance(query, m.embedding), - })) - .sort((a, b) => a.distance - b.distance) - .slice(0, limit); - return scored.map(s => s.ref); + private getWeekMonday(date: Date = new Date()): string { + const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate())); + const day = d.getUTCDay(); + const diff = day === 0 ? -6 : 1 - day; + d.setUTCDate(d.getUTCDate() + diff); + return d.toISOString().slice(0, 10); } private listNodes(memories: Memory[]): MemoryRef[] { return memories.map(m => ({name: m.name, description: m.description})); } - async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise { - const conversation = history - .filter(h => h.role === 'user' || h.role === 'assistant') - .map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim(); - if (!conversation) return []; - - const uid = `${Date.now()}_${Math.random().toString(36).slice(2)}`; - // NOTE: adjust field names below (id/tool_call_id/name) to match your LLMMessage/tool-call schema. - const pending = {role: 'tool', name: 'memory_process', id: uid, content: 'Processing…'} as unknown as LLMMessage; - history.push(pending); - - const mem = this.unwrap(memories); - const buckets = await this.factAgent(conversation, mem, options, getWeekMonday()); - const touched: Memory[] = []; - - for (const {subject, facts} of buckets) { - let node = mem.find(m => m.name === subject); - if (!node) { - node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []}; - mem.push(node); - } - this.appendFacts(node, facts); - const [e] = await this.llm.embedding(node.content); - if (e) node.embedding = e.embedding; - this.touch(node.name); - touched.push(node); - } - - if (touched.length) { - rebuildGraph(mem); - this.sync(memories); - (pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`; - for (const node of touched) this.reconcile(node, memories, options).catch(() => {}); - } else { - (pending as any).content = 'Nothing worth remembering.'; - } - - return touched; - } - - /** Manual/cron entry point. scope 'touched' only reconciles docs with a pending Facts inbox. */ - async reconcileVault(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise { - const mem = this.unwrap(memories); - const targets = scope === 'all' ? mem : mem.filter(m => m.content.includes(FACTS_HEADING)); - await Promise.all(targets.map(node => this.reconcile(node, memories, options))); - this.sync(memories); - } - - /** - * Coalescing queue: if a doc is already reconciling, mark it dirty and abort the in-flight - * request. The loop below always re-reads node.content fresh, so nothing is ever dropped. - */ private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise { const key = node.name; const existing = this.queues.get(key); @@ -402,21 +341,20 @@ export class MemoryManager { const entry = {dirty: false, request: null, task: Promise.resolve()}; this.queues.set(key, entry); - const mem = this.unwrap(memories); + const store = this.access(memories); entry.task = (async () => { do { entry.dirty = false; - await this.reconcileDoc(node, mem, options, entry); + await this.docAgent(node, store.list, options, entry); } while (entry.dirty); })().finally(() => { this.queues.delete(key); - rebuildGraph(mem); - this.sync(memories); + store.commit(); }); return entry.task; } - private async reconcileDoc(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise { + private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise { const currentBody = this.stripHeader(node.content); let update; try { @@ -470,50 +408,128 @@ ${currentBody} if (e) node.embedding = e.embedding; } - private async factAgent(conversation: string, memories: Memory[], options: LLMRequest, weekKey: string): Promise { - const buckets = new Map(); - const ghosts = this.ghostNodes(memories); + private parseFrontmatter(content: string): {fm: Map, body: string} { + const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); + if (!match) return {fm: new Map(), body: content}; + const fm = new Map(); + for (const line of match[1].split('\n')) { + const i = line.indexOf(':'); + if (i === -1) continue; + fm.set(line.slice(0, i).trim(), line.slice(i + 1).trim()); + } + return {fm, body: match[2]}; + } - await this.llm.ask(conversation, { - model: options.model, - temperature: 0.2, - system: `You are a fact extractor. Analyze this conversation and extract facts worth remembering long-term. + private stripHeader(content: string): string { + return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart(); + } -Rules: -- ONLY extract current facts the USER explicitly stated about themselves, their work, or their projects -- ONLY extract decisions that were MADE during this conversation -- DO NOT extract anything the AI said, its capabilities, or meta-conversation about the AI -- DO NOT extract greetings, pleasantries, or generic exchanges -- DO NOT extract deltas or changes in facts; ONLY the end fact -- If nothing worth remembering was said, do not call any tools + private touchHeader(node: Memory, body: string): string { + const {fm} = this.parseFrontmatter(node.content); + fm.set('name', node.name); + fm.set('description', node.description || ''); + fm.set('modified', new Date().toISOString()); + return this.writeFrontmatter(fm, body); + } -When extracting facts, you MUST also decide the exact destination path: -- Reuse node names (including ghost) as much as possible IF the facts belongs there -- All information primarily about the user should go under "People/User" -- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide) — you are not limited to any fixed list of collections, use whatever fits -- For journal entries, use "Journal" + private writeFrontmatter(fm: Map, body: string): string { + const lines = [...fm.entries()].map(([k, v]) => `${k}: ${v}`); + return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`; + } -Available nodes: -- Journal -${this.listNodes(memories).filter(n => !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'} -${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`, - tools: [{ - name: 'facts_extract', - description: 'Submit facts with their destination', - args: { - destination: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true}, - facts: {type: 'string', description: 'Comma-separated facts', required: true}, - }, - fn: (args: any) => { - const subject = args.destination.trim().toLowerCase() === 'journal' - ? `Journal/${weekKey}` : args.destination.trim(); - const facts = buckets.get(subject) ?? []; - facts.push(...dedupeFacts(String(args.facts).split(','))); - buckets.set(subject, facts); - return 'Recorded'; - }, - }], - }); - return buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})); + decay() { + for (const [name, ttl] of this.recentlyTouched) { + if (ttl <= 1) this.recentlyTouched.delete(name); + else this.recentlyTouched.set(name, ttl - 1); + } + } + + touch(name: string, ttl = 2) { + this.recentlyTouched.set(name, ttl); + } + + forget(name: string, memories: Memory[] | MemoryCache): boolean { + return this.access(memories).forget(name); + } + + async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise { + const store = this.access(memories); + if (!store.list.length) return []; + + await store.backfillEmbeddings(this.llm); + + const [e] = await this.llm.embedding(query); + if (!e) return []; + + const vectorResults = store.search(e.embedding, limit); + const found = new Set(vectorResults.map(r => r.name)); + + if (graphDepth > 0) { + let frontier = [...found]; + for (let depth = 0; depth < graphDepth && frontier.length; depth++) { + const next: string[] = []; + for (const name of frontier) { + const node = store.find(name); + if (!node) continue; + for (const link of node.links) { + if (!found.has(link) && store.find(link)) { + found.add(link); + next.push(link); + } + } + } + frontier = next; + } + } + + const vectorOrder = vectorResults.map(r => r.name); + const graphExpansions = [...found].filter(n => !vectorOrder.includes(n)); + return [...vectorOrder, ...graphExpansions].map(n => store.find(n)!).filter(Boolean); + } + + async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise { + const conversation = history + .filter(h => h.role === 'user' || h.role === 'assistant') + .map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim(); + if (!conversation) return []; + + const uid = `${Date.now()}_${Math.random().toString(36).slice(2)}`; + const pending = {role: 'tool', name: 'memory_process', id: uid, content: conversation} as unknown as LLMMessage; + history.push(pending); + + const store = this.access(memories); + const buckets = await this.factAgent(conversation, store, options, this.getWeekMonday()); + const touched: Memory[] = []; + + for (const {subject, facts} of buckets) { + let node = store.find(subject); + if (!node) { + node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []}; + store.list.push(node); + } + this.appendFacts(node, facts); + const [e] = await this.llm.embedding(node.content); + if (e) node.embedding = e.embedding; + this.touch(node.name); + touched.push(node); + } + + if (touched.length) { + store.commit(); + (pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`; + await Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {}))); + } else { + (pending as any).content = 'Nothing worth remembering.'; + } + + (touched as any).uid = uid; + return touched; + } + + async reconcileVault(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise { + const store = this.access(memories); + const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(FACTS_HEADING)); + await Promise.all(targets.map(node => this.reconcile(node, memories, options))); + store.commit(); } } diff --git a/tests/llm.spec.ts b/tests/llm.spec.ts deleted file mode 100644 index 2e887cd..0000000 --- a/tests/llm.spec.ts +++ /dev/null @@ -1,167 +0,0 @@ - -import {describe, it, expect, vi, beforeEach} from 'vitest'; -import LLM from '../src/llm'; - -const {FakeProvider, providerLog} = vi.hoisted(() => { - const providerLog: any[] = []; - class FakeProvider { - model: string; - constructor(...args: any[]) { this.model = args[args.length - 1]; } - ask(message: string, opts: any) { - let aborted = false; - const p = (async () => { - const script = (globalThis as any).__scripts?.[this.model]; - const plan = script ? script(message, opts) : {text: ''}; - providerLog.push({model: this.model, message, system: opts.system, tools: (opts.tools || []).map((t: any) => t.name)}); - for (const c of plan.calls || []) { - if (aborted) break; - const tool = (opts.tools || []).find((t: any) => t.name === c.tool); - const id = c.id || `${c.tool}_${Math.random()}`; - const content = await tool.fn(c.args, opts.stream, null, id); - opts.history.push({role: 'tool', id, name: c.tool, args: c.args, content, timestamp: Date.now()}); - } - const text = plan.text ?? ''; - if (opts.stream && text) opts.stream({text, done: true}); - opts.history.push({role: 'assistant', content: text, timestamp: Date.now(), duration: 10, tps: 5}); - return text; - })(); - return Object.assign(p, {abort: () => { aborted = true; }}); - } - } - return {FakeProvider, providerLog}; -}); - -vi.mock('../src/antrhopic.ts', () => ({Anthropic: FakeProvider})); -vi.mock('../src/open-ai.ts', () => ({OpenAi: FakeProvider})); - -function makeAi(models: any) { - return {options: {llm: {models}}} as any; -} - -beforeEach(() => { - providerLog.length = 0; - (globalThis as any).__scripts = {}; -}); - -describe('LLM cross-provider interchangeability', () => { - it('runs identical tool calls the same way on an anthropic-backed model and an openai-backed model', async () => { - const ai = makeAi({ - claude: {proto: 'anthropic', token: 'x'}, - gpt: {proto: 'openai', token: 'y', host: 'http://local'}, - }); - const llm = new LLM(ai); - const calc = { - name: 'calc_add', - description: 'Add two numbers', - args: {a: {type: 'number', required: true}, b: {type: 'number', required: true}}, - fn: (args: any) => String(args.a + args.b), - }; - - (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'}); - (globalThis as any).__scripts.gpt = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'}); - - const historyA: any[] = [], historyB: any[] = []; - const respA = await llm.ask('add 2 and 3', {model: 'claude', tools: [calc], history: historyA}); - const respB = await llm.ask('add 2 and 3', {model: 'gpt', tools: [calc], history: historyB}); - - expect(respA).toBe('Result: 5'); - expect(respB).toBe('Result: 5'); - expect(providerLog.find(l => l.model === 'claude')!.tools).toContain('calc_add'); - expect(providerLog.find(l => l.model === 'gpt')!.tools).toContain('calc_add'); - - // tool timing gets recomputed from real execution regardless of proto - for (const h of [historyA.find(h => h.name === 'calc_add'), historyB.find(h => h.name === 'calc_add')]) { - expect(h.content).toBe('5'); - expect(typeof h.duration).toBe('number'); - expect(typeof h.tps).toBe('number'); - } - }); - - it('lets the same shared history flow across model + proto swaps with different system prompts', async () => { - const ai = makeAi({ - claude: {proto: 'anthropic', token: 'x'}, - gpt: {proto: 'openai', token: 'y', host: 'http://local'}, - }); - const llm = new LLM(ai); - const history: any[] = []; - - (globalThis as any).__scripts.claude = () => ({text: 'Hi from claude'}); - (globalThis as any).__scripts.gpt = () => ({text: 'Hi from gpt'}); - - const r1 = await llm.ask('hello', {model: 'claude', system: 'You are terse.', history}); - const r2 = await llm.ask('follow up', {model: 'gpt', system: 'You are verbose.', history}); - - expect(r1).toBe('Hi from claude'); - expect(r2).toBe('Hi from gpt'); - expect(history.filter(h => h.role === 'assistant').map(h => h.content)).toEqual(['Hi from claude', 'Hi from gpt']); - expect(providerLog[0].system).toContain('You are terse.'); - expect(providerLog[1].system).toContain('You are verbose.'); - }); - - it('exposes MCP tools the same way no matter which proto backs the model', async () => { - const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); - const llm = new LLM(ai); - const mcp = [{name: 'weather', host: 'http://mcp.local'}]; - - global.fetch = vi.fn(async (url: string, opts?: any) => { - if (url.endsWith('/tools')) { - return {json: async () => ({tools: [{name: 'lookup', description: 'Look up weather', inputSchema: {properties: {city: {type: 'string'}}, required: ['city']}}]})} as any; - } - const body = JSON.parse(opts.body); - return {json: async () => ({content: [{text: `Sunny in ${body.arguments.city}`}]})} as any; - }) as any; - - for (const model of ['claude', 'gpt']) { - (globalThis as any).__scripts[model] = () => ({calls: [{tool: 'weather_lookup', args: {city: 'Rome'}}], text: 'done'}); - const history: any[] = []; - await llm.ask('weather?', {model, mcp, history}); - expect(history.find(h => h.name === 'weather_lookup')?.content).toBe('Sunny in Rome'); - } - }); - - it('exposes and resolves skill documents identically across protos', async () => { - const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); - const llm = new LLM(ai); - const skills = [{name: 'Onboarding', description: 'How to onboard a user', content: 'Step 1...'}]; - - for (const model of ['claude', 'gpt']) { - (globalThis as any).__scripts[model] = () => ({calls: [{tool: 'skill_read', args: {name: 'Onboarding'}}], text: 'done'}); - const history: any[] = []; - await llm.ask('onboard me', {model, skills, history}); - expect(history.find(h => h.name === 'skill_read')?.content).toContain('Step 1...'); - } - }); - - it('delegate agent mutates the shared history directly and backfills the orchestrator response, across protos', async () => { - const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); - const llm = new LLM(ai); - const history: any[] = [{role: 'user', content: 'research quantum computing'}]; - const researcher = {name: 'researcher', system: 'You research topics.', delegate: true, model: 'gpt'}; - - (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'agent_researcher', args: {}}], text: ''}); - (globalThis as any).__scripts.gpt = () => ({text: 'Quantum computers use qubits.'}); - - const resp = await llm.ask('go', {model: 'claude', agents: [researcher], history}); - - expect(resp).toBe('Quantum computers use qubits.'); - expect(history.some(h => h.role === 'assistant' && h.content === 'Quantum computers use qubits.')).toBe(true); - expect(history.find(h => h.name === 'agent_researcher')?.content).toBe(''); - }); - - it('regular (non-delegate) subagent keeps its own isolated history separate from the parent, across protos', async () => { - const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); - const llm = new LLM(ai); - const history: any[] = []; - const summarizer = {name: 'summarizer', system: 'You summarize text.', model: 'gpt'}; - - (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'subagent_summarizer', args: {context: 'a long article', instructions: 'summarize it'}}], text: 'Summary: short version'}); - (globalThis as any).__scripts.gpt = () => ({text: 'short version'}); - - const resp = await llm.ask('summarize this', {model: 'claude', agents: [summarizer], history}); - - expect(resp).toBe('Summary: short version'); - expect(history.find(h => h.name === 'subagent_summarizer')?.content).toBe('short version'); - // isolated history - subagent's own assistant turn never leaks into the parent - expect(history.some(h => h.role === 'assistant' && h.content === 'short version')).toBe(false); - }); -}); diff --git a/tests/memory.spec.ts b/tests/memory.spec.ts deleted file mode 100644 index 6e2da0c..0000000 --- a/tests/memory.spec.ts +++ /dev/null @@ -1,256 +0,0 @@ -import {describe, it, expect, vi, beforeEach} from 'vitest'; -import {MemoryManager, MemoryCache, rebuildGraph, Memory} from '../src/memory'; - -function makeMemory(overrides: Partial = {}): Memory { - return { - name: 'Test/Doc', - description: '', - content: '', - embedding: [], - links: [], - backlinks: [], - ...overrides, - }; -} - -function makeLLM() { - return { - embedding: vi.fn(async (_text: string) => [{embedding: [1, 0, 0]}]), - ask: vi.fn(async () => undefined), - }; -} - -describe('rebuildGraph', () => { - it('extracts [[WikiLinks]] from content, excluding self-links', () => { - const a = makeMemory({name: 'A', content: '[[B]] and [[A]] and [[C]]'}); - const b = makeMemory({name: 'B', content: 'no links here'}); - const mem = [a, b]; - - rebuildGraph(mem); - - expect(a.links).toEqual(['B', 'C']); - expect(b.links).toEqual([]); - }); - - it('computes backlinks only for links that resolve to a real node', () => { - const a = makeMemory({name: 'A', content: '[[B]] [[Missing]]'}); - const b = makeMemory({name: 'B', content: ''}); - const mem = [a, b]; - - rebuildGraph(mem); - - expect(b.backlinks).toEqual(['A']); - expect(mem.find(m => m.name === 'Missing')).toBeUndefined(); - }); - - it('resets stale backlinks on every rebuild (no leftover from a removed link)', () => { - const a = makeMemory({name: 'A', content: '[[B]]'}); - const b = makeMemory({name: 'B', content: ''}); - const mem = [a, b]; - rebuildGraph(mem); - expect(b.backlinks).toEqual(['A']); - - a.content = 'no more links'; - rebuildGraph(mem); - expect(b.backlinks).toEqual([]); - }); -}); - -describe('MemoryCache', () => { - it('finds nearest neighbor by embedding via KD-tree search', () => { - const close = makeMemory({name: 'Close', embedding: [1, 0, 0]}); - const far = makeMemory({name: 'Far', embedding: [0, 0, 1]}); - const cache = new MemoryCache([close, far]); - - const results = cache.search([1, 0, 0], 1); - - expect(results[0].name).toBe('Close'); - }); - - it('rebuilds the tree on add/update/remove', () => { - const cache = new MemoryCache([makeMemory({name: 'A', embedding: [1, 0, 0]})]); - cache.add(makeMemory({name: 'B', embedding: [0, 1, 0]})); - expect(cache.search([0, 1, 0], 1)[0].name).toBe('B'); - - cache.remove('B'); - expect(cache.search([0, 1, 0], 1)[0]?.name).not.toBe('B'); - }); -}); - -describe('MemoryManager.forget', () => { - it('removes the node and recomputes backlinks for the rest of the graph', () => { - const llm = makeLLM(); - const mgr = new MemoryManager(llm); - const a = makeMemory({name: 'A', content: '[[B]]'}); - const b = makeMemory({name: 'B', content: '[[C]]'}); - const c = makeMemory({name: 'C', content: ''}); - const mem = [a, b, c]; - rebuildGraph(mem); - expect(c.backlinks).toEqual(['B']); - - const ok = mgr.forget('B', mem); - - expect(ok).toBe(true); - expect(mem.find(m => m.name === 'B')).toBeUndefined(); - expect(a.links).toEqual(['B']); - expect(c.backlinks).toEqual([]); - }); - - it('returns false for an unknown name', () => { - const mgr = new MemoryManager(makeLLM()); - expect(mgr.forget('Nope', [makeMemory({name: 'A'})])).toBe(false); - }); -}); - -describe('MemoryManager.recollect', () => { - it('orders vector matches first, then expands one hop via links', async () => { - const llm = makeLLM(); - llm.embedding.mockResolvedValue([{embedding: [1, 0, 0]}]); - const mgr = new MemoryManager(llm); - - const near = makeMemory({name: 'Near', embedding: [1, 0, 0], content: '[[Linked]]'}); - const linked = makeMemory({name: 'Linked', embedding: [0, 0, 1], content: ''}); - const far = makeMemory({name: 'Far', embedding: [0, 1, 0], content: ''}); - const mem = [near, linked, far]; - rebuildGraph(mem); - - const result = await mgr.recollect('query', mem, 1, 1); - - expect(result.map(r => r.name)).toEqual(['Near', 'Linked']); - }); - - it('returns [] when there are no memories', async () => { - const mgr = new MemoryManager(makeLLM()); - expect(await mgr.recollect('q', [])).toEqual([]); - }); -}); - -describe('MemoryManager.memorize (fast path)', () => { - let llm: ReturnType; - let mgr: MemoryManager; - - beforeEach(() => { - llm = makeLLM(); - mgr = new MemoryManager(llm); - }); - - it('pushes a pending tool message, then resolves it to links once facts land', async () => { - llm.ask.mockImplementation(async (_prompt: string, opts: any) => { - if (opts.tools) { - opts.tools[0].fn({destination: 'Projects/Oxide', facts: 'Uses a hybrid memory system'}); - return undefined; - } - return {description: 'd', content: '# doc'}; - }); - - const history: any[] = [{role: 'user', content: 'we use a hybrid memory system'}]; - const touched = await mgr.memorize(history, [], {model: 'test'} as any); - - const pending = history.find(h => h.name === 'memory_process'); - expect(pending).toBeDefined(); - expect(pending.content).toContain('[[Projects/Oxide]]'); - expect(touched.map(t => t.name)).toEqual(['Projects/Oxide']); - }); - - it('creates a new node and appends facts under "## Facts" without calling the doc LLM', async () => { - llm.ask.mockImplementation(async (_prompt: string, opts: any) => { - if (opts.tools) opts.tools[0].fn({destination: 'People/Sarah', facts: 'Works at Acme, Likes hiking'}); - return undefined; - }); - - const mem: Memory[] = []; - await mgr.memorize([{role: 'user', content: 'Sarah works at Acme and likes hiking'}] as any, mem, {model: 'test'} as any); - - const node = mem.find(m => m.name === 'People/Sarah')!; - expect(node).toBeDefined(); - expect(node.content).toContain('## Facts'); - expect(node.content).toContain('- Works at Acme'); - expect(node.content).toContain('- Likes hiking'); - // doc reconciler LLM (schema call) should NOT have been awaited synchronously in this fast path assertion - }); - - it('routes "journal" destination to Journal/{weekMonday}', async () => { - llm.ask.mockImplementation(async (_prompt: string, opts: any) => { - if (opts.tools) opts.tools[0].fn({destination: 'journal', facts: 'Shipped v1'}); - return undefined; - }); - - const mem: Memory[] = []; - const touched = await mgr.memorize([{role: 'user', content: 'shipped v1 today'}] as any, mem, {model: 'test'} as any); - - expect(touched[0].name).toMatch(/^Journal\/\d{4}-\d{2}-\d{2}$/); - }); - - it('reports nothing to remember when no facts are extracted', async () => { - llm.ask.mockResolvedValue(undefined); // tools present but fn never called - - const history: any[] = [{role: 'user', content: 'hey'}]; - const touched = await mgr.memorize(history, [], {model: 'test'} as any); - - expect(touched).toEqual([]); - expect(history.find(h => h.name === 'memory_process').content).toBe('Nothing worth remembering.'); - }); - - it('returns [] and does nothing for an empty conversation', async () => { - const touched = await mgr.memorize([], [], {model: 'test'} as any); - expect(touched).toEqual([]); - expect(llm.ask).not.toHaveBeenCalled(); - }); -}); - -describe('MemoryManager reconcileVault', () => { - it('integrates the "## Facts" section via the doc LLM and removes it', async () => { - const llm = makeLLM(); - llm.ask.mockResolvedValue({description: 'Tidy summary', content: '# Doc\n\nIntegrated fact.'}); - const mgr = new MemoryManager(llm); - - const node = makeMemory({ - name: 'Projects/Oxide', - content: '---\nname: Projects/Oxide\n---\n\n# Doc\n\n## Facts\n- some raw fact\n', - }); - const mem = [node]; - - await mgr.reconcileVault(mem, {model: 'test'} as any, 'all'); - - expect(node.content).not.toContain('## Facts'); - expect(node.content).toContain('Integrated fact.'); - expect(node.description).toBe('Tidy summary'); - }); - - it('only targets docs with a pending Facts inbox when scope is "touched"', async () => { - const llm = makeLLM(); - llm.ask.mockResolvedValue({description: 'd', content: '# clean'}); - const mgr = new MemoryManager(llm); - - const dirty = makeMemory({name: 'A', content: '## Facts\n- x'}); - const clean = makeMemory({name: 'B', content: '# already tidy'}); - await mgr.reconcileVault([dirty, clean], {model: 'test'} as any, 'touched'); - - expect(dirty.content).toContain('# clean'); // rewritten (frontmatter now wraps it) - expect(clean.content).toBe('# already tidy'); // untouched, never queued - }); -}); - -describe('MemoryManager reconcile coalescing', () => { - it('coalesces a second call while one is in-flight: marks dirty, aborts, reuses the same task promise', () => { - const llm = makeLLM(); - const abort = vi.fn(); - let calls = 0; - llm.ask.mockImplementation(() => { - calls++; - const pending: any = new Promise(() => {}); // never resolves in this test - pending.abort = abort; - return pending; - }); - const mgr: any = new MemoryManager(llm); - const node = makeMemory({name: 'Q', content: '# Q\n\n## Facts\n- f'}); - const mem = [node]; - - const p1 = mgr.reconcile(node, mem, {model: 'test'}); - const p2 = mgr.reconcile(node, mem, {model: 'test'}); - - expect(p2).toBe(p1); // same in-flight task, not a new queue entry - expect(abort).toHaveBeenCalledTimes(1); // second call aborted the in-flight request - expect(calls).toBe(1); // no second ask() fired synchronously — it'll rerun via the dirty loop - }); -});