import {describe, it, expect, vi, beforeEach} from 'vitest'; import LLM from '../src/llm'; const {FakeProvider, providerLog} = vi.hoisted(() => { const providerLog: any[] = []; class FakeProvider { model: string; constructor(...args: any[]) { this.model = args[args.length - 1]; } ask(message: string, opts: any) { let aborted = false; const p = (async () => { const script = (globalThis as any).__scripts?.[this.model]; const plan = script ? script(message, opts) : {text: ''}; providerLog.push({model: this.model, message, system: opts.system, tools: (opts.tools || []).map((t: any) => t.name)}); for (const c of plan.calls || []) { if (aborted) break; const tool = (opts.tools || []).find((t: any) => t.name === c.tool); const id = c.id || `${c.tool}_${Math.random()}`; const content = await tool.fn(c.args, opts.stream, null, id); opts.history.push({role: 'tool', id, name: c.tool, args: c.args, content, timestamp: Date.now()}); } const text = plan.text ?? ''; if (opts.stream && text) opts.stream({text, done: true}); opts.history.push({role: 'assistant', content: text, timestamp: Date.now(), duration: 10, tps: 5}); return text; })(); return Object.assign(p, {abort: () => { aborted = true; }}); } } return {FakeProvider, providerLog}; }); vi.mock('../src/antrhopic.ts', () => ({Anthropic: FakeProvider})); vi.mock('../src/open-ai.ts', () => ({OpenAi: FakeProvider})); function makeAi(models: any) { return {options: {llm: {models}}} as any; } beforeEach(() => { providerLog.length = 0; (globalThis as any).__scripts = {}; }); describe('LLM cross-provider interchangeability', () => { it('runs identical tool calls the same way on an anthropic-backed model and an openai-backed model', async () => { const ai = makeAi({ claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}, }); const llm = new LLM(ai); const calc = { name: 'calc_add', description: 'Add two numbers', args: {a: {type: 'number', required: true}, b: {type: 'number', required: true}}, fn: (args: any) => String(args.a + args.b), }; (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'}); (globalThis as any).__scripts.gpt = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'}); const historyA: any[] = [], historyB: any[] = []; const respA = await llm.ask('add 2 and 3', {model: 'claude', tools: [calc], history: historyA}); const respB = await llm.ask('add 2 and 3', {model: 'gpt', tools: [calc], history: historyB}); expect(respA).toBe('Result: 5'); expect(respB).toBe('Result: 5'); expect(providerLog.find(l => l.model === 'claude')!.tools).toContain('calc_add'); expect(providerLog.find(l => l.model === 'gpt')!.tools).toContain('calc_add'); // tool timing gets recomputed from real execution regardless of proto for (const h of [historyA.find(h => h.name === 'calc_add'), historyB.find(h => h.name === 'calc_add')]) { expect(h.content).toBe('5'); expect(typeof h.duration).toBe('number'); expect(typeof h.tps).toBe('number'); } }); it('lets the same shared history flow across model + proto swaps with different system prompts', async () => { const ai = makeAi({ claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}, }); const llm = new LLM(ai); const history: any[] = []; (globalThis as any).__scripts.claude = () => ({text: 'Hi from claude'}); (globalThis as any).__scripts.gpt = () => ({text: 'Hi from gpt'}); const r1 = await llm.ask('hello', {model: 'claude', system: 'You are terse.', history}); const r2 = await llm.ask('follow up', {model: 'gpt', system: 'You are verbose.', history}); expect(r1).toBe('Hi from claude'); expect(r2).toBe('Hi from gpt'); expect(history.filter(h => h.role === 'assistant').map(h => h.content)).toEqual(['Hi from claude', 'Hi from gpt']); expect(providerLog[0].system).toContain('You are terse.'); expect(providerLog[1].system).toContain('You are verbose.'); }); it('exposes MCP tools the same way no matter which proto backs the model', async () => { const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); const llm = new LLM(ai); const mcp = [{name: 'weather', host: 'http://mcp.local'}]; global.fetch = vi.fn(async (url: string, opts?: any) => { if (url.endsWith('/tools')) { return {json: async () => ({tools: [{name: 'lookup', description: 'Look up weather', inputSchema: {properties: {city: {type: 'string'}}, required: ['city']}}]})} as any; } const body = JSON.parse(opts.body); return {json: async () => ({content: [{text: `Sunny in ${body.arguments.city}`}]})} as any; }) as any; for (const model of ['claude', 'gpt']) { (globalThis as any).__scripts[model] = () => ({calls: [{tool: 'weather_lookup', args: {city: 'Rome'}}], text: 'done'}); const history: any[] = []; await llm.ask('weather?', {model, mcp, history}); expect(history.find(h => h.name === 'weather_lookup')?.content).toBe('Sunny in Rome'); } }); it('exposes and resolves skill documents identically across protos', async () => { const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); const llm = new LLM(ai); const skills = [{name: 'Onboarding', description: 'How to onboard a user', content: 'Step 1...'}]; for (const model of ['claude', 'gpt']) { (globalThis as any).__scripts[model] = () => ({calls: [{tool: 'skill_read', args: {name: 'Onboarding'}}], text: 'done'}); const history: any[] = []; await llm.ask('onboard me', {model, skills, history}); expect(history.find(h => h.name === 'skill_read')?.content).toContain('Step 1...'); } }); it('delegate agent mutates the shared history directly and backfills the orchestrator response, across protos', async () => { const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); const llm = new LLM(ai); const history: any[] = [{role: 'user', content: 'research quantum computing'}]; const researcher = {name: 'researcher', system: 'You research topics.', delegate: true, model: 'gpt'}; (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'agent_researcher', args: {}}], text: ''}); (globalThis as any).__scripts.gpt = () => ({text: 'Quantum computers use qubits.'}); const resp = await llm.ask('go', {model: 'claude', agents: [researcher], history}); expect(resp).toBe('Quantum computers use qubits.'); expect(history.some(h => h.role === 'assistant' && h.content === 'Quantum computers use qubits.')).toBe(true); expect(history.find(h => h.name === 'agent_researcher')?.content).toBe(''); }); it('regular (non-delegate) subagent keeps its own isolated history separate from the parent, across protos', async () => { const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}}); const llm = new LLM(ai); const history: any[] = []; const summarizer = {name: 'summarizer', system: 'You summarize text.', model: 'gpt'}; (globalThis as any).__scripts.claude = () => ({calls: [{tool: 'subagent_summarizer', args: {context: 'a long article', instructions: 'summarize it'}}], text: 'Summary: short version'}); (globalThis as any).__scripts.gpt = () => ({text: 'short version'}); const resp = await llm.ask('summarize this', {model: 'claude', agents: [summarizer], history}); expect(resp).toBe('Summary: short version'); expect(history.find(h => h.name === 'subagent_summarizer')?.content).toBe('short version'); // isolated history - subagent's own assistant turn never leaks into the parent expect(history.some(h => h.role === 'assistant' && h.content === 'short version')).toBe(false); }); });