168 lines
7.8 KiB
TypeScript
168 lines
7.8 KiB
TypeScript
|
|
import {describe, it, expect, vi, beforeEach} from 'vitest';
|
|
import LLM from '../src/llm';
|
|
|
|
const {FakeProvider, providerLog} = vi.hoisted(() => {
|
|
const providerLog: any[] = [];
|
|
class FakeProvider {
|
|
model: string;
|
|
constructor(...args: any[]) { this.model = args[args.length - 1]; }
|
|
ask(message: string, opts: any) {
|
|
let aborted = false;
|
|
const p = (async () => {
|
|
const script = (globalThis as any).__scripts?.[this.model];
|
|
const plan = script ? script(message, opts) : {text: ''};
|
|
providerLog.push({model: this.model, message, system: opts.system, tools: (opts.tools || []).map((t: any) => t.name)});
|
|
for (const c of plan.calls || []) {
|
|
if (aborted) break;
|
|
const tool = (opts.tools || []).find((t: any) => t.name === c.tool);
|
|
const id = c.id || `${c.tool}_${Math.random()}`;
|
|
const content = await tool.fn(c.args, opts.stream, null, id);
|
|
opts.history.push({role: 'tool', id, name: c.tool, args: c.args, content, timestamp: Date.now()});
|
|
}
|
|
const text = plan.text ?? '';
|
|
if (opts.stream && text) opts.stream({text, done: true});
|
|
opts.history.push({role: 'assistant', content: text, timestamp: Date.now(), duration: 10, tps: 5});
|
|
return text;
|
|
})();
|
|
return Object.assign(p, {abort: () => { aborted = true; }});
|
|
}
|
|
}
|
|
return {FakeProvider, providerLog};
|
|
});
|
|
|
|
vi.mock('../src/antrhopic.ts', () => ({Anthropic: FakeProvider}));
|
|
vi.mock('../src/open-ai.ts', () => ({OpenAi: FakeProvider}));
|
|
|
|
function makeAi(models: any) {
|
|
return {options: {llm: {models}}} as any;
|
|
}
|
|
|
|
beforeEach(() => {
|
|
providerLog.length = 0;
|
|
(globalThis as any).__scripts = {};
|
|
});
|
|
|
|
describe('LLM cross-provider interchangeability', () => {
|
|
it('runs identical tool calls the same way on an anthropic-backed model and an openai-backed model', async () => {
|
|
const ai = makeAi({
|
|
claude: {proto: 'anthropic', token: 'x'},
|
|
gpt: {proto: 'openai', token: 'y', host: 'http://local'},
|
|
});
|
|
const llm = new LLM(ai);
|
|
const calc = {
|
|
name: 'calc_add',
|
|
description: 'Add two numbers',
|
|
args: {a: {type: 'number', required: true}, b: {type: 'number', required: true}},
|
|
fn: (args: any) => String(args.a + args.b),
|
|
};
|
|
|
|
(globalThis as any).__scripts.claude = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'});
|
|
(globalThis as any).__scripts.gpt = () => ({calls: [{tool: 'calc_add', args: {a: 2, b: 3}}], text: 'Result: 5'});
|
|
|
|
const historyA: any[] = [], historyB: any[] = [];
|
|
const respA = await llm.ask('add 2 and 3', {model: 'claude', tools: [calc], history: historyA});
|
|
const respB = await llm.ask('add 2 and 3', {model: 'gpt', tools: [calc], history: historyB});
|
|
|
|
expect(respA).toBe('Result: 5');
|
|
expect(respB).toBe('Result: 5');
|
|
expect(providerLog.find(l => l.model === 'claude')!.tools).toContain('calc_add');
|
|
expect(providerLog.find(l => l.model === 'gpt')!.tools).toContain('calc_add');
|
|
|
|
// tool timing gets recomputed from real execution regardless of proto
|
|
for (const h of [historyA.find(h => h.name === 'calc_add'), historyB.find(h => h.name === 'calc_add')]) {
|
|
expect(h.content).toBe('5');
|
|
expect(typeof h.duration).toBe('number');
|
|
expect(typeof h.tps).toBe('number');
|
|
}
|
|
});
|
|
|
|
it('lets the same shared history flow across model + proto swaps with different system prompts', async () => {
|
|
const ai = makeAi({
|
|
claude: {proto: 'anthropic', token: 'x'},
|
|
gpt: {proto: 'openai', token: 'y', host: 'http://local'},
|
|
});
|
|
const llm = new LLM(ai);
|
|
const history: any[] = [];
|
|
|
|
(globalThis as any).__scripts.claude = () => ({text: 'Hi from claude'});
|
|
(globalThis as any).__scripts.gpt = () => ({text: 'Hi from gpt'});
|
|
|
|
const r1 = await llm.ask('hello', {model: 'claude', system: 'You are terse.', history});
|
|
const r2 = await llm.ask('follow up', {model: 'gpt', system: 'You are verbose.', history});
|
|
|
|
expect(r1).toBe('Hi from claude');
|
|
expect(r2).toBe('Hi from gpt');
|
|
expect(history.filter(h => h.role === 'assistant').map(h => h.content)).toEqual(['Hi from claude', 'Hi from gpt']);
|
|
expect(providerLog[0].system).toContain('You are terse.');
|
|
expect(providerLog[1].system).toContain('You are verbose.');
|
|
});
|
|
|
|
it('exposes MCP tools the same way no matter which proto backs the model', async () => {
|
|
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
|
const llm = new LLM(ai);
|
|
const mcp = [{name: 'weather', host: 'http://mcp.local'}];
|
|
|
|
global.fetch = vi.fn(async (url: string, opts?: any) => {
|
|
if (url.endsWith('/tools')) {
|
|
return {json: async () => ({tools: [{name: 'lookup', description: 'Look up weather', inputSchema: {properties: {city: {type: 'string'}}, required: ['city']}}]})} as any;
|
|
}
|
|
const body = JSON.parse(opts.body);
|
|
return {json: async () => ({content: [{text: `Sunny in ${body.arguments.city}`}]})} as any;
|
|
}) as any;
|
|
|
|
for (const model of ['claude', 'gpt']) {
|
|
(globalThis as any).__scripts[model] = () => ({calls: [{tool: 'weather_lookup', args: {city: 'Rome'}}], text: 'done'});
|
|
const history: any[] = [];
|
|
await llm.ask('weather?', {model, mcp, history});
|
|
expect(history.find(h => h.name === 'weather_lookup')?.content).toBe('Sunny in Rome');
|
|
}
|
|
});
|
|
|
|
it('exposes and resolves skill documents identically across protos', async () => {
|
|
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
|
const llm = new LLM(ai);
|
|
const skills = [{name: 'Onboarding', description: 'How to onboard a user', content: 'Step 1...'}];
|
|
|
|
for (const model of ['claude', 'gpt']) {
|
|
(globalThis as any).__scripts[model] = () => ({calls: [{tool: 'skill_read', args: {name: 'Onboarding'}}], text: 'done'});
|
|
const history: any[] = [];
|
|
await llm.ask('onboard me', {model, skills, history});
|
|
expect(history.find(h => h.name === 'skill_read')?.content).toContain('Step 1...');
|
|
}
|
|
});
|
|
|
|
it('delegate agent mutates the shared history directly and backfills the orchestrator response, across protos', async () => {
|
|
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
|
const llm = new LLM(ai);
|
|
const history: any[] = [{role: 'user', content: 'research quantum computing'}];
|
|
const researcher = {name: 'researcher', system: 'You research topics.', delegate: true, model: 'gpt'};
|
|
|
|
(globalThis as any).__scripts.claude = () => ({calls: [{tool: 'agent_researcher', args: {}}], text: ''});
|
|
(globalThis as any).__scripts.gpt = () => ({text: 'Quantum computers use qubits.'});
|
|
|
|
const resp = await llm.ask('go', {model: 'claude', agents: [researcher], history});
|
|
|
|
expect(resp).toBe('Quantum computers use qubits.');
|
|
expect(history.some(h => h.role === 'assistant' && h.content === 'Quantum computers use qubits.')).toBe(true);
|
|
expect(history.find(h => h.name === 'agent_researcher')?.content).toBe('');
|
|
});
|
|
|
|
it('regular (non-delegate) subagent keeps its own isolated history separate from the parent, across protos', async () => {
|
|
const ai = makeAi({claude: {proto: 'anthropic', token: 'x'}, gpt: {proto: 'openai', token: 'y', host: 'http://local'}});
|
|
const llm = new LLM(ai);
|
|
const history: any[] = [];
|
|
const summarizer = {name: 'summarizer', system: 'You summarize text.', model: 'gpt'};
|
|
|
|
(globalThis as any).__scripts.claude = () => ({calls: [{tool: 'subagent_summarizer', args: {context: 'a long article', instructions: 'summarize it'}}], text: 'Summary: short version'});
|
|
(globalThis as any).__scripts.gpt = () => ({text: 'short version'});
|
|
|
|
const resp = await llm.ask('summarize this', {model: 'claude', agents: [summarizer], history});
|
|
|
|
expect(resp).toBe('Summary: short version');
|
|
expect(history.find(h => h.name === 'subagent_summarizer')?.content).toBe('short version');
|
|
// isolated history - subagent's own assistant turn never leaks into the parent
|
|
expect(history.some(h => h.role === 'assistant' && h.content === 'short version')).toBe(false);
|
|
});
|
|
});
|