Pass deligate subagents full history, improved memory managment
This commit is contained in:
167
tests/llm.spec.ts
Normal file
167
tests/llm.spec.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
|
||||
import {describe, it, expect, vi, beforeEach} from 'vitest';
|
||||
import LLM from '../src/llm';
|
||||
|
||||
const {FakeProvider, providerLog} = vi.hoisted(() => {
|
||||
const providerLog: any[] = [];
|
||||
class FakeProvider {
|
||||
model: string;
|
||||
constructor(...args: any[]) { this.model = args[args.length - 1]; }
|
||||
ask(message: string, opts: any) {
|
||||
let aborted = false;
|
||||
const p = (async () => {
|
||||
const script = (globalThis as any).__scripts?.[this.model];
|
||||
const plan = script ? script(message, opts) : {text: ''};
|
||||
providerLog.push({model: this.model, message, system: opts.system, tools: (opts.tools || []).map((t: any) => t.name)});
|
||||
for (const c of plan.calls || []) {
|
||||
if (aborted) break;
|
||||
const tool = (opts.tools || []).find((t: any) => t.name === c.tool);
|
||||
const id = c.id || `${c.tool}_${Math.random()}`;
|
||||
const content = await tool.fn(c.args, opts.stream, null, id);
|
||||
opts.history.push({role: 'tool', id, name: c.tool, args: c.args, content, timestamp: Date.now()});
|
||||
}
|
||||
const text = plan.text ?? '';
|
||||
if (opts.stream && text) opts.stream({text, done: true});
|
||||
opts.history.push({role: 'assistant', content: text, timestamp: Date.now(), duration: 10, tps: 5});
|
||||
return text;
|
||||
})();
|
||||
return Object.assign(p, {abort: () => { aborted = true; }});
|
||||
}
|
||||
}
|
||||
return {FakeProvider, providerLog};
|
||||
});
|
||||
|
||||
vi.mock('../src/antrhopic.ts', () => ({Anthropic: FakeProvider}));
|
||||
vi.mock('../src/open-ai.ts', () => ({OpenAi: FakeProvider}));
|
||||
|
||||
function makeAi(models: any) {
|
||||
return {options: {llm: {models}}} as any;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
providerLog.length = 0;
|
||||
(globalThis as any).__scripts = {};
|
||||
});
|
||||
|
||||
describe('LLM cross-provider interchangeability', () => {
|
||||
it('runs identical tool calls the same way on an anthropic-backed model and an openai-backed model', async () => {
|
||||
const ai = makeAi({
|
||||
claude: {proto: 'anthropic', token: 'x'},
|
||||
gpt: {proto: 'openai', token: 'y', host: 'http://local'},
|
||||
});
|
||||
const llm = new LLM(ai);
|
||||
const calc = {
|
||||
name: 'calc_add',
|
||||
description: 'Add two numbers',
|
||||
args: {a: {type: 'number', required: true}, b: {type: 'number', required: true}},
|
||||
fn: (args: any) => String(args.a + args.b),
|
||||
};
|
||||
|
||||
(globalThis as any).__scripts.claude = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'});
|
||||
(globalThis as any).__scripts.gpt = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'});
|
||||
|
||||
const historyA: any[] = [], historyB: any[] = [];
|
||||
const respA = await llm.ask('add 2 and 3', {model: 'claude', tools: [calc], history: historyA});
|
||||
const respB = await llm.ask('add 2 and 3', {model: 'gpt', tools: [calc], history: historyB});
|
||||
|
||||
expect(respA).toBe('Result: 5');
|
||||
expect(respB).toBe('Result: 5');
|
||||
expect(providerLog.find(l => l.model === 'claude')!.tools).toContain('calc_add');
|
||||
expect(providerLog.find(l => l.model === 'gpt')!.tools).toContain('calc_add');
|
||||
|
||||
// tool timing gets recomputed from real execution regardless of proto
|
||||
for (const h of [historyA.find(h => h.name === 'calc_add'), historyB.find(h => h.name === 'calc_add')]) {
|
||||
expect(h.content).toBe('5');
|
||||
expect(typeof h.duration).toBe('number');
|
||||
expect(typeof h.tps).toBe('number');
|
||||
}
|
||||
});
|
||||
|
||||
it('lets the same shared history flow across model + proto swaps with different system prompts', async () => {
|
||||
const ai = makeAi({
|
||||
claude: {proto: 'anthropic', token: 'x'},
|
||||
gpt: {proto: 'openai', token: 'y', host: 'http://local'},
|
||||
});
|
||||
const llm = new LLM(ai);
|
||||
const history: any[] = [];
|
||||
|
||||
(globalThis as any).__scripts.claude = () => ({text: 'Hi from claude'});
|
||||
(globalThis as any).__scripts.gpt = () => ({text: 'Hi from gpt'});
|
||||
|
||||
const r1 = await llm.ask('hello', {model: 'claude', system: 'You are terse.', history});
|
||||
const r2 = await llm.ask('follow up', {model: 'gpt', system: 'You are verbose.', history});
|
||||
|
||||
expect(r1).toBe('Hi from claude');
|
||||
expect(r2).toBe('Hi from gpt');
|
||||
expect(history.filter(h => h.role === 'assistant').map(h => h.content)).toEqual(['Hi from claude', 'Hi from gpt']);
|
||||
expect(providerLog[0].system).toContain('You are terse.');
|
||||
expect(providerLog[1].system).toContain('You are verbose.');
|
||||
});
|
||||
|
||||
it('exposes MCP tools the same way no matter which proto backs the model', async () => {
|
||||
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
||||
const llm = new LLM(ai);
|
||||
const mcp = [{name: 'weather', host: 'http://mcp.local'}];
|
||||
|
||||
global.fetch = vi.fn(async (url: string, opts?: any) => {
|
||||
if (url.endsWith('/tools')) {
|
||||
return {json: async () => ({tools: [{name: 'lookup', description: 'Look up weather', inputSchema: {properties: {city: {type: 'string'}}, required: ['city']}}]})} as any;
|
||||
}
|
||||
const body = JSON.parse(opts.body);
|
||||
return {json: async () => ({content: [{text: `Sunny in ${body.arguments.city}`}]})} as any;
|
||||
}) as any;
|
||||
|
||||
for (const model of ['claude', 'gpt']) {
|
||||
(globalThis as any).__scripts[model] = () => ({calls: [{tool: 'weather_lookup', args: {city: 'Rome'}}], text: 'done'});
|
||||
const history: any[] = [];
|
||||
await llm.ask('weather?', {model, mcp, history});
|
||||
expect(history.find(h => h.name === 'weather_lookup')?.content).toBe('Sunny in Rome');
|
||||
}
|
||||
});
|
||||
|
||||
it('exposes and resolves skill documents identically across protos', async () => {
|
||||
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
||||
const llm = new LLM(ai);
|
||||
const skills = [{name: 'Onboarding', description: 'How to onboard a user', content: 'Step 1...'}];
|
||||
|
||||
for (const model of ['claude', 'gpt']) {
|
||||
(globalThis as any).__scripts[model] = () => ({calls: [{tool: 'skill_read', args: {name: 'Onboarding'}}], text: 'done'});
|
||||
const history: any[] = [];
|
||||
await llm.ask('onboard me', {model, skills, history});
|
||||
expect(history.find(h => h.name === 'skill_read')?.content).toContain('Step 1...');
|
||||
}
|
||||
});
|
||||
|
||||
it('delegate agent mutates the shared history directly and backfills the orchestrator response, across protos', async () => {
|
||||
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
||||
const llm = new LLM(ai);
|
||||
const history: any[] = [{role: 'user', content: 'research quantum computing'}];
|
||||
const researcher = {name: 'researcher', system: 'You research topics.', delegate: true, model: 'gpt'};
|
||||
|
||||
(globalThis as any).__scripts.claude = () => ({calls: [{tool: 'agent_researcher', args: {}}], text: ''});
|
||||
(globalThis as any).__scripts.gpt = () => ({text: 'Quantum computers use qubits.'});
|
||||
|
||||
const resp = await llm.ask('go', {model: 'claude', agents: [researcher], history});
|
||||
|
||||
expect(resp).toBe('Quantum computers use qubits.');
|
||||
expect(history.some(h => h.role === 'assistant' && h.content === 'Quantum computers use qubits.')).toBe(true);
|
||||
expect(history.find(h => h.name === 'agent_researcher')?.content).toBe('');
|
||||
});
|
||||
|
||||
it('regular (non-delegate) subagent keeps its own isolated history separate from the parent, across protos', async () => {
|
||||
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
||||
const llm = new LLM(ai);
|
||||
const history: any[] = [];
|
||||
const summarizer = {name: 'summarizer', system: 'You summarize text.', model: 'gpt'};
|
||||
|
||||
(globalThis as any).__scripts.claude = () => ({calls: [{tool: 'subagent_summarizer', args: {context: 'a long article', instructions: 'summarize it'}}], text: 'Summary: short version'});
|
||||
(globalThis as any).__scripts.gpt = () => ({text: 'short version'});
|
||||
|
||||
const resp = await llm.ask('summarize this', {model: 'claude', agents: [summarizer], history});
|
||||
|
||||
expect(resp).toBe('Summary: short version');
|
||||
expect(history.find(h => h.name === 'subagent_summarizer')?.content).toBe('short version');
|
||||
// isolated history - subagent's own assistant turn never leaks into the parent
|
||||
expect(history.some(h => h.role === 'assistant' && h.content === 'short version')).toBe(false);
|
||||
});
|
||||
});
|
||||
256
tests/memory.spec.ts
Normal file
256
tests/memory.spec.ts
Normal file
@@ -0,0 +1,256 @@
|
||||
import {describe, it, expect, vi, beforeEach} from 'vitest';
|
||||
import {MemoryManager, MemoryCache, rebuildGraph, Memory} from '../src/memory';
|
||||
|
||||
function makeMemory(overrides: Partial<Memory> = {}): Memory {
|
||||
return {
|
||||
name: 'Test/Doc',
|
||||
description: '',
|
||||
content: '',
|
||||
embedding: [],
|
||||
links: [],
|
||||
backlinks: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeLLM() {
|
||||
return {
|
||||
embedding: vi.fn(async (_text: string) => [{embedding: [1, 0, 0]}]),
|
||||
ask: vi.fn(async () => undefined),
|
||||
};
|
||||
}
|
||||
|
||||
describe('rebuildGraph', () => {
|
||||
it('extracts [[WikiLinks]] from content, excluding self-links', () => {
|
||||
const a = makeMemory({name: 'A', content: '[[B]] and [[A]] and [[C]]'});
|
||||
const b = makeMemory({name: 'B', content: 'no links here'});
|
||||
const mem = [a, b];
|
||||
|
||||
rebuildGraph(mem);
|
||||
|
||||
expect(a.links).toEqual(['B', 'C']);
|
||||
expect(b.links).toEqual([]);
|
||||
});
|
||||
|
||||
it('computes backlinks only for links that resolve to a real node', () => {
|
||||
const a = makeMemory({name: 'A', content: '[[B]] [[Missing]]'});
|
||||
const b = makeMemory({name: 'B', content: ''});
|
||||
const mem = [a, b];
|
||||
|
||||
rebuildGraph(mem);
|
||||
|
||||
expect(b.backlinks).toEqual(['A']);
|
||||
expect(mem.find(m => m.name === 'Missing')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('resets stale backlinks on every rebuild (no leftover from a removed link)', () => {
|
||||
const a = makeMemory({name: 'A', content: '[[B]]'});
|
||||
const b = makeMemory({name: 'B', content: ''});
|
||||
const mem = [a, b];
|
||||
rebuildGraph(mem);
|
||||
expect(b.backlinks).toEqual(['A']);
|
||||
|
||||
a.content = 'no more links';
|
||||
rebuildGraph(mem);
|
||||
expect(b.backlinks).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryCache', () => {
|
||||
it('finds nearest neighbor by embedding via KD-tree search', () => {
|
||||
const close = makeMemory({name: 'Close', embedding: [1, 0, 0]});
|
||||
const far = makeMemory({name: 'Far', embedding: [0, 0, 1]});
|
||||
const cache = new MemoryCache([close, far]);
|
||||
|
||||
const results = cache.search([1, 0, 0], 1);
|
||||
|
||||
expect(results[0].name).toBe('Close');
|
||||
});
|
||||
|
||||
it('rebuilds the tree on add/update/remove', () => {
|
||||
const cache = new MemoryCache([makeMemory({name: 'A', embedding: [1, 0, 0]})]);
|
||||
cache.add(makeMemory({name: 'B', embedding: [0, 1, 0]}));
|
||||
expect(cache.search([0, 1, 0], 1)[0].name).toBe('B');
|
||||
|
||||
cache.remove('B');
|
||||
expect(cache.search([0, 1, 0], 1)[0]?.name).not.toBe('B');
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryManager.forget', () => {
|
||||
it('removes the node and recomputes backlinks for the rest of the graph', () => {
|
||||
const llm = makeLLM();
|
||||
const mgr = new MemoryManager(llm);
|
||||
const a = makeMemory({name: 'A', content: '[[B]]'});
|
||||
const b = makeMemory({name: 'B', content: '[[C]]'});
|
||||
const c = makeMemory({name: 'C', content: ''});
|
||||
const mem = [a, b, c];
|
||||
rebuildGraph(mem);
|
||||
expect(c.backlinks).toEqual(['B']);
|
||||
|
||||
const ok = mgr.forget('B', mem);
|
||||
|
||||
expect(ok).toBe(true);
|
||||
expect(mem.find(m => m.name === 'B')).toBeUndefined();
|
||||
expect(a.links).toEqual(['B']);
|
||||
expect(c.backlinks).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns false for an unknown name', () => {
|
||||
const mgr = new MemoryManager(makeLLM());
|
||||
expect(mgr.forget('Nope', [makeMemory({name: 'A'})])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryManager.recollect', () => {
|
||||
it('orders vector matches first, then expands one hop via links', async () => {
|
||||
const llm = makeLLM();
|
||||
llm.embedding.mockResolvedValue([{embedding: [1, 0, 0]}]);
|
||||
const mgr = new MemoryManager(llm);
|
||||
|
||||
const near = makeMemory({name: 'Near', embedding: [1, 0, 0], content: '[[Linked]]'});
|
||||
const linked = makeMemory({name: 'Linked', embedding: [0, 0, 1], content: ''});
|
||||
const far = makeMemory({name: 'Far', embedding: [0, 1, 0], content: ''});
|
||||
const mem = [near, linked, far];
|
||||
rebuildGraph(mem);
|
||||
|
||||
const result = await mgr.recollect('query', mem, 1, 1);
|
||||
|
||||
expect(result.map(r => r.name)).toEqual(['Near', 'Linked']);
|
||||
});
|
||||
|
||||
it('returns [] when there are no memories', async () => {
|
||||
const mgr = new MemoryManager(makeLLM());
|
||||
expect(await mgr.recollect('q', [])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryManager.memorize (fast path)', () => {
|
||||
let llm: ReturnType<typeof makeLLM>;
|
||||
let mgr: MemoryManager;
|
||||
|
||||
beforeEach(() => {
|
||||
llm = makeLLM();
|
||||
mgr = new MemoryManager(llm);
|
||||
});
|
||||
|
||||
it('pushes a pending tool message, then resolves it to links once facts land', async () => {
|
||||
llm.ask.mockImplementation(async (_prompt: string, opts: any) => {
|
||||
if (opts.tools) {
|
||||
opts.tools[0].fn({destination: 'Projects/Oxide', facts: 'Uses a hybrid memory system'});
|
||||
return undefined;
|
||||
}
|
||||
return {description: 'd', content: '# doc'};
|
||||
});
|
||||
|
||||
const history: any[] = [{role: 'user', content: 'we use a hybrid memory system'}];
|
||||
const touched = await mgr.memorize(history, [], {model: 'test'} as any);
|
||||
|
||||
const pending = history.find(h => h.name === 'memory_process');
|
||||
expect(pending).toBeDefined();
|
||||
expect(pending.content).toContain('[[Projects/Oxide]]');
|
||||
expect(touched.map(t => t.name)).toEqual(['Projects/Oxide']);
|
||||
});
|
||||
|
||||
it('creates a new node and appends facts under "## Facts" without calling the doc LLM', async () => {
|
||||
llm.ask.mockImplementation(async (_prompt: string, opts: any) => {
|
||||
if (opts.tools) opts.tools[0].fn({destination: 'People/Sarah', facts: 'Works at Acme, Likes hiking'});
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const mem: Memory[] = [];
|
||||
await mgr.memorize([{role: 'user', content: 'Sarah works at Acme and likes hiking'}] as any, mem, {model: 'test'} as any);
|
||||
|
||||
const node = mem.find(m => m.name === 'People/Sarah')!;
|
||||
expect(node).toBeDefined();
|
||||
expect(node.content).toContain('## Facts');
|
||||
expect(node.content).toContain('- Works at Acme');
|
||||
expect(node.content).toContain('- Likes hiking');
|
||||
// doc reconciler LLM (schema call) should NOT have been awaited synchronously in this fast path assertion
|
||||
});
|
||||
|
||||
it('routes "journal" destination to Journal/{weekMonday}', async () => {
|
||||
llm.ask.mockImplementation(async (_prompt: string, opts: any) => {
|
||||
if (opts.tools) opts.tools[0].fn({destination: 'journal', facts: 'Shipped v1'});
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const mem: Memory[] = [];
|
||||
const touched = await mgr.memorize([{role: 'user', content: 'shipped v1 today'}] as any, mem, {model: 'test'} as any);
|
||||
|
||||
expect(touched[0].name).toMatch(/^Journal\/\d{4}-\d{2}-\d{2}$/);
|
||||
});
|
||||
|
||||
it('reports nothing to remember when no facts are extracted', async () => {
|
||||
llm.ask.mockResolvedValue(undefined); // tools present but fn never called
|
||||
|
||||
const history: any[] = [{role: 'user', content: 'hey'}];
|
||||
const touched = await mgr.memorize(history, [], {model: 'test'} as any);
|
||||
|
||||
expect(touched).toEqual([]);
|
||||
expect(history.find(h => h.name === 'memory_process').content).toBe('Nothing worth remembering.');
|
||||
});
|
||||
|
||||
it('returns [] and does nothing for an empty conversation', async () => {
|
||||
const touched = await mgr.memorize([], [], {model: 'test'} as any);
|
||||
expect(touched).toEqual([]);
|
||||
expect(llm.ask).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryManager reconcileVault', () => {
|
||||
it('integrates the "## Facts" section via the doc LLM and removes it', async () => {
|
||||
const llm = makeLLM();
|
||||
llm.ask.mockResolvedValue({description: 'Tidy summary', content: '# Doc\n\nIntegrated fact.'});
|
||||
const mgr = new MemoryManager(llm);
|
||||
|
||||
const node = makeMemory({
|
||||
name: 'Projects/Oxide',
|
||||
content: '---\nname: Projects/Oxide\n---\n\n# Doc\n\n## Facts\n- some raw fact\n',
|
||||
});
|
||||
const mem = [node];
|
||||
|
||||
await mgr.reconcileVault(mem, {model: 'test'} as any, 'all');
|
||||
|
||||
expect(node.content).not.toContain('## Facts');
|
||||
expect(node.content).toContain('Integrated fact.');
|
||||
expect(node.description).toBe('Tidy summary');
|
||||
});
|
||||
|
||||
it('only targets docs with a pending Facts inbox when scope is "touched"', async () => {
|
||||
const llm = makeLLM();
|
||||
llm.ask.mockResolvedValue({description: 'd', content: '# clean'});
|
||||
const mgr = new MemoryManager(llm);
|
||||
|
||||
const dirty = makeMemory({name: 'A', content: '## Facts\n- x'});
|
||||
const clean = makeMemory({name: 'B', content: '# already tidy'});
|
||||
await mgr.reconcileVault([dirty, clean], {model: 'test'} as any, 'touched');
|
||||
|
||||
expect(dirty.content).toContain('# clean'); // rewritten (frontmatter now wraps it)
|
||||
expect(clean.content).toBe('# already tidy'); // untouched, never queued
|
||||
});
|
||||
});
|
||||
|
||||
describe('MemoryManager reconcile coalescing', () => {
|
||||
it('coalesces a second call while one is in-flight: marks dirty, aborts, reuses the same task promise', () => {
|
||||
const llm = makeLLM();
|
||||
const abort = vi.fn();
|
||||
let calls = 0;
|
||||
llm.ask.mockImplementation(() => {
|
||||
calls++;
|
||||
const pending: any = new Promise(() => {}); // never resolves in this test
|
||||
pending.abort = abort;
|
||||
return pending;
|
||||
});
|
||||
const mgr: any = new MemoryManager(llm);
|
||||
const node = makeMemory({name: 'Q', content: '# Q\n\n## Facts\n- f'});
|
||||
const mem = [node];
|
||||
|
||||
const p1 = mgr.reconcile(node, mem, {model: 'test'});
|
||||
const p2 = mgr.reconcile(node, mem, {model: 'test'});
|
||||
|
||||
expect(p2).toBe(p1); // same in-flight task, not a new queue entry
|
||||
expect(abort).toHaveBeenCalledTimes(1); // second call aborted the in-flight request
|
||||
expect(calls).toBe(1); // no second ask() fired synchronously — it'll rerun via the dirty loop
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user