456 lines
17 KiB
TypeScript
456 lines
17 KiB
TypeScript
import {AbortablePromise, Ai} from './ai.ts';
|
|
import {Anthropic} from './antrhopic.ts';
|
|
import {OpenAi} from './open-ai.ts';
|
|
import {LLMProvider} from './provider.ts';
|
|
import {AiTool, AiToolArg} from './tools.ts';
|
|
import {fileURLToPath} from 'url';
|
|
import {dirname, join} from 'path';
|
|
import {spawn} from 'node:child_process';
|
|
import {Memory, MemoryCache, MemoryManager} from './memory.ts';
|
|
|
|
export type AnthropicConfig = {proto: 'anthropic', token: string};
|
|
export type OpenAiConfig = {proto: 'openai', host?: string, token: string};
|
|
|
|
export type LLMMessage = {
|
|
/** Message originator */
|
|
role: 'assistant' | 'system' | 'user';
|
|
/** Message content */
|
|
content: string | any;
|
|
/** Timestamp */
|
|
timestamp?: number;
|
|
} | {
|
|
/** Tool call */
|
|
role: 'tool';
|
|
/** Unique ID for call */
|
|
id: string;
|
|
/** Tool that was run */
|
|
name: string;
|
|
/** Tool arguments */
|
|
args: any;
|
|
/** Tool result */
|
|
content: undefined | string;
|
|
/** Tool error */
|
|
error?: undefined | string;
|
|
/** Timestamp */
|
|
timestamp?: number;
|
|
}
|
|
|
|
export type LLMRequest = {
|
|
/** Return a parsed JSON object that matches the schema */
|
|
schema?: AiToolArg;
|
|
/** System prompt */
|
|
system?: string;
|
|
/** Message history */
|
|
history?: LLMMessage[];
|
|
/** Max tokens for request */
|
|
max_tokens?: number;
|
|
/** 0 = Rigid Logic, 1 = Balanced, 2 = Hyper Creative **/
|
|
temperature?: number;
|
|
/** Available tools */
|
|
tools?: AiTool[];
|
|
/** LLM model */
|
|
model?: string;
|
|
/** Stream response */
|
|
stream?: (chunk: {text?: string, tool?: string, done?: true}) => any;
|
|
/** Compress old messages in the chat to free up context */
|
|
compress?: {max: number; min: number};
|
|
/** User's memory documents - RAG injected automatically each turn */
|
|
memory?: Memory[] | MemoryCache;
|
|
/** Model to use for memory operations */
|
|
memoryModel?: string;
|
|
/** Skill documents the AI can browse and read on demand */
|
|
skills?: Skill[];
|
|
/** MCP servers to connect and expose as tools */
|
|
mcp?: McpServer[];
|
|
}
|
|
|
|
export type McpServer = {
|
|
/** MCP server name for humans */
|
|
name: string;
|
|
/** Host URL */
|
|
host: string;
|
|
/** Server access token */
|
|
token?: string;
|
|
}
|
|
|
|
export type Skill = {
|
|
/** Name of skill for humans */
|
|
name: string;
|
|
/** Description LLM will use to decide to learn a skill */
|
|
description: string;
|
|
/** Skill instructions */
|
|
content: string;
|
|
}
|
|
|
|
|
|
class LLM {
|
|
private memoryManager!: MemoryManager;
|
|
|
|
defaultModel!: string;
|
|
models: {[model: string]: LLMProvider} = {};
|
|
|
|
constructor(public readonly ai: Ai) {
|
|
if(!ai.options.llm?.models) return;
|
|
Object.entries(ai.options.llm.models).forEach(([model, config]) => {
|
|
if(!this.defaultModel) this.defaultModel = model;
|
|
if(config.proto == 'anthropic') this.models[model] = new Anthropic(this.ai, config.token, model);
|
|
else if(config.proto == 'openai') this.models[model] = new OpenAi(this.ai, config.host || null, config.token, model);
|
|
});
|
|
this.memoryManager = new MemoryManager(this);
|
|
}
|
|
|
|
private async setupMcp(servers: McpServer[] = []): Promise<{prompt: string, tools: AiTool[]}> {
|
|
if(!servers?.length) return {prompt: '', tools: []};
|
|
const allTools: AiTool[] = [];
|
|
await Promise.all(servers.map(async server => {
|
|
const res = await fetch(`${server.host}/tools`, {headers: server.token ? {Authorization: `Bearer ${server.token}`} : {}});
|
|
const mcp: any = await res.json();
|
|
if(!mcp?.tools) return;
|
|
for(const t of mcp.tools) {
|
|
const args: Record<string, any> = {};
|
|
if(t.inputSchema?.properties) {
|
|
for(const [key, val] of Object.entries<any>(t.inputSchema.properties)) {
|
|
args[key] = {type: val.type || 'string', description: val.description || '', required: t.inputSchema.required?.includes(key)};
|
|
}
|
|
}
|
|
allTools.push({
|
|
name: `${server.name}_${t.name}`,
|
|
description: t.description || '',
|
|
args,
|
|
fn: async (a: any) => {
|
|
const r = await fetch(`${server.host}/tools/call`, {
|
|
method: 'POST',
|
|
headers: {'Content-Type': 'application/json', ...(server.token ? {Authorization: `Bearer ${server.token}`} : {})},
|
|
body: JSON.stringify({name: t.name, arguments: a})
|
|
});
|
|
const data: any = await r.json();
|
|
return data?.content?.[0]?.text ?? JSON.stringify(data);
|
|
}
|
|
});
|
|
}
|
|
}));
|
|
|
|
const list = allTools.map(t => `- ${t.name}: ${t.description}`).join('\n');
|
|
return {
|
|
prompt: `You have access to the following MCP tools:\n${list}`,
|
|
tools: allTools
|
|
};
|
|
}
|
|
|
|
private setupSkills(skills: Skill[] = []): {prompt: string, tools: AiTool[]} {
|
|
if(!skills?.length) return {prompt: '', tools: []};
|
|
const list = skills.map(s => `- ${s.name}: ${s.description}`).join('\n');
|
|
return {
|
|
prompt: `You have access to the following skill documents, use \`read_skill\` to access them:\n${list}`,
|
|
tools: [{
|
|
name: 'read_skill',
|
|
description: 'Read the full content of a skill/knowledge document',
|
|
args: {
|
|
name: {type: 'string', description: 'Exact skill name', required: true}
|
|
},
|
|
fn: (args: any) => {
|
|
const skill = skills.find(s => s.name === args.name);
|
|
if(!skill) return `Skill not found. Available:\n${list}`;
|
|
return `# ${skill.name}\n${skill.content}`;
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
|
|
ask(message: string, options: LLMRequest = {}): AbortablePromise<string> {
|
|
options = <any>{
|
|
system: '',
|
|
...this.ai.options.llm,
|
|
models: undefined,
|
|
history: [],
|
|
...options,
|
|
}
|
|
const m = options.model || this.defaultModel;
|
|
if(!this.models[m]) throw new Error(`Model does not exist: ${m}`);
|
|
let abort = () => {};
|
|
return Object.assign(new Promise<string>(async res => {
|
|
let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
|
|
const prompts: string[] = [];
|
|
let history = options.history || [];
|
|
|
|
// MCP
|
|
const mcp = options.mcp || this.ai.options?.llm?.mcp;
|
|
if(mcp?.length) {
|
|
const m = await this.setupMcp(mcp);
|
|
prompts.unshift(m.prompt);
|
|
tools.push(...m.tools);
|
|
}
|
|
|
|
// Skills
|
|
const skills = options.skills || this.ai.options?.llm?.skills;
|
|
if(skills?.length) {
|
|
const s = this.setupSkills(skills);
|
|
prompts.unshift(s.prompt);
|
|
tools.push(...s.tools);
|
|
}
|
|
|
|
// Memory
|
|
if (options.memory) {
|
|
const mems = options.memory instanceof MemoryCache ? options.memory.memories : options.memory;
|
|
const relevant = await this.memoryManager.recollect(message, options.memory, 5);
|
|
prompts.unshift(`You have access to the following memory files:
|
|
${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')}
|
|
${relevant.length ? `
|
|
Relevant memories have been preloaded:
|
|
${relevant.map(r => `
|
|
**${r.name}**
|
|
${r.description}
|
|
${r.content}
|
|
`).join('\n---\n')}
|
|
` : ''}`.trim());
|
|
tools.push(this.memoryManager.tools.read(options.memory));
|
|
}
|
|
|
|
prompts.unshift(options.system || this.ai.options.llm?.system || '');
|
|
const resp = await this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
|
|
|
|
// Trim memory injections from history
|
|
if(options.memory) {
|
|
history.splice(0, history.length, ...history.filter(h => h.role !== 'tool' || h.name !== 'recall'));
|
|
}
|
|
|
|
// Auto-memorize before compressing
|
|
if(options.compress && this.estimateTokens(history) >= options.compress.max) {
|
|
if(options.memory) await this.memoryManager.memorize(history, options.memory, {model: options.memoryModel || this.defaultModel, ...options});
|
|
const compressed = await this.compressHistory(history, options.compress.max, options.compress.min, options);
|
|
if(options.history) options.history.splice(0, options.history.length, ...compressed);
|
|
}
|
|
|
|
return res(resp);
|
|
}), {abort});
|
|
}
|
|
|
|
/**
|
|
* Digest full conversation history into memory documents.
|
|
* Call on session end to persist the conversation.
|
|
*/
|
|
async updateMemory(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest = {}): Promise<Memory[]> {
|
|
return this.memoryManager.memorize(history, memories, {model: this.defaultModel, ...options});
|
|
}
|
|
|
|
/**
|
|
* Compress chat history to reduce context size
|
|
* @param {LLMMessage[]} history Chatlog that will be compressed
|
|
* @param max Trigger compression once context is larger than max
|
|
* @param min Leave messages less than the token minimum, summarize the rest
|
|
* @param {LLMRequest} options LLM options
|
|
* @returns {Promise<LLMMessage[]>} New chat history will summary at index 0
|
|
*/
|
|
async compressHistory(history: LLMMessage[], max: number, min: number, options?: LLMRequest): Promise<LLMMessage[]> {
|
|
if(this.estimateTokens(history) < max) return history;
|
|
let keep = 0, tokens = 0;
|
|
for(let m of history.toReversed()) {
|
|
tokens += this.estimateTokens(m.content);
|
|
if(tokens < min) keep++;
|
|
else break;
|
|
}
|
|
if(history.length <= keep) return history;
|
|
const system = history[0].role == 'system' ? history[0] : null,
|
|
recent = keep == 0 ? [] : history.slice(-keep),
|
|
process = (keep == 0 ? history : history.slice(0, -keep)).filter(h => h.role === 'assistant' || h.role === 'user');
|
|
|
|
const summary: any = await this.summarize(process.map(m => `[${m.role}]: ${m.content}`).join('\n\n'), 500, options);
|
|
const d = Date.now();
|
|
const h = [{role: <any>'tool', name: 'summary', id: `summary_` + d, args: {}, content: `Conversation Summary: ${summary?.summary}`, timestamp: d}, ...recent];
|
|
if(system) h.splice(0, 0, system);
|
|
return h;
|
|
}
|
|
|
|
/**
|
|
* Compare the difference between embeddings (calculates the angle between two vectors)
|
|
* @param {number[]} v1 First embedding / vector comparison
|
|
* @param {number[]} v2 Second embedding / vector for comparison
|
|
* @returns {number} Similarity values 0-1: 0 = unique, 1 = identical
|
|
*/
|
|
cosineSimilarity(v1: number[], v2: number[]): number {
|
|
if (v1.length !== v2.length) throw new Error('Vectors must be same length');
|
|
let dotProduct = 0, normA = 0, normB = 0;
|
|
for (let i = 0; i < v1.length; i++) {
|
|
dotProduct += v1[i] * v2[i];
|
|
normA += v1[i] * v1[i];
|
|
normB += v2[i] * v2[i];
|
|
}
|
|
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
|
return denominator === 0 ? 0 : dotProduct / denominator;
|
|
}
|
|
|
|
/**
|
|
* Chunk text into parts for AI digestion
|
|
* @param {object | string} target Item that will be chunked (objects get converted)
|
|
* @param {number} maxTokens Chunking size. More = better context, less = more specific (Search by paragraphs or lines)
|
|
* @param {number} overlapTokens Includes previous X tokens to provide continuity to AI (In addition to max tokens)
|
|
* @returns {string[]} Chunked strings
|
|
*/
|
|
chunk(target: object | string, maxTokens = 500, overlapTokens = 50): string[] {
|
|
const objString = (obj: any, path = ''): string[] => {
|
|
if(!obj) return [];
|
|
return Object.entries(obj).flatMap(([key, value]) => {
|
|
const p = path ? `${path}${isNaN(+key) ? `.${key}` : `[${key}]`}` : key;
|
|
if(typeof value === 'object' && !Array.isArray(value)) return objString(value, p);
|
|
return `${p}: ${Array.isArray(value) ? value.join(', ') : value}`;
|
|
});
|
|
};
|
|
const lines = typeof target === 'object' ? objString(target) : target.toString().split('\n');
|
|
const tokens = lines.flatMap(l => [...l.split(/\s+/).filter(Boolean), '\n']);
|
|
const chunks: string[] = [];
|
|
for(let i = 0; i < tokens.length;) {
|
|
let text = '', j = i;
|
|
while(j < tokens.length) {
|
|
const next = text + (text ? ' ' : '') + tokens[j];
|
|
if(this.estimateTokens(next.replace(/\s*\n\s*/g, '\n')) > maxTokens && text) break;
|
|
text = next;
|
|
j++;
|
|
}
|
|
const clean = text.replace(/\s*\n\s*/g, '\n').trim();
|
|
if(clean) chunks.push(clean);
|
|
i = Math.max(j - overlapTokens, j === i ? i + 1 : j);
|
|
}
|
|
return chunks;
|
|
}
|
|
|
|
/**
|
|
* Create a vector representation of a string
|
|
* @param {object | string} target Item that will be embedded (objects get converted)
|
|
* @param {maxTokens?: number, overlapTokens?: number} opts Options for embedding such as chunk sizes
|
|
* @returns {Promise<Awaited<{index: number, embedding: number[], text: string, tokens: number}>[]>} Chunked embeddings
|
|
*/
|
|
embedding(target: object | string, opts: {maxTokens?: number, overlapTokens?: number} = {}): AbortablePromise<{index: number, embedding: number[], text: string, tokens: number}[]> {
|
|
let {maxTokens = 500, overlapTokens = 50} = opts;
|
|
let aborted = false;
|
|
const abort = () => { aborted = true; };
|
|
|
|
const embed = (text: string): Promise<number[]> => {
|
|
return new Promise((resolve, reject) => {
|
|
if(aborted) return reject(new Error('Aborted'));
|
|
const args: string[] = [
|
|
join(dirname(fileURLToPath(import.meta.url)), 'embedder.js'),
|
|
<string>this.ai.options.path,
|
|
this.ai.options?.embedder || 'bge-small-en-v1.5'
|
|
];
|
|
const proc = spawn('node', args, {stdio: ['pipe', 'pipe', 'ignore']});
|
|
proc.stdin.write(text);
|
|
proc.stdin.end();
|
|
let output = '';
|
|
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
|
proc.on('close', (code: number) => {
|
|
if(aborted) return reject(new Error('Aborted'));
|
|
if(code === 0) {
|
|
try {
|
|
const result = JSON.parse(output);
|
|
resolve(result.embedding);
|
|
} catch(err) {
|
|
reject(err);
|
|
}
|
|
} else {
|
|
reject(new Error(`Embedder process exited with code ${code}`));
|
|
}
|
|
});
|
|
proc.on('error', reject);
|
|
});
|
|
};
|
|
|
|
const p = (async () => {
|
|
const chunks = this.chunk(target, maxTokens, overlapTokens), results: any[] = [];
|
|
for(let i = 0; i < chunks.length; i++) {
|
|
if(aborted) break;
|
|
const text = chunks[i];
|
|
const embedding = await embed(text);
|
|
results.push({index: i, embedding, text, tokens: this.estimateTokens(text)});
|
|
}
|
|
return results;
|
|
})();
|
|
return <any>Object.assign(p, {abort});
|
|
}
|
|
|
|
/**
|
|
* Estimate variable as tokens
|
|
* @param history Object to size
|
|
* @returns {number} Rough token count
|
|
*/
|
|
estimateTokens(history: any): number {
|
|
const text = JSON.stringify(history);
|
|
return Math.ceil((text.length / 4) * 1.2);
|
|
}
|
|
|
|
/**
|
|
* Compare the difference between two strings using tensor math
|
|
* @param target Text that will be checked
|
|
* @param {string} searchTerms Multiple search terms to check against target
|
|
* @returns {{avg: number, max: number, similarities: number[]}} Similarity values 0-1: 0 = unique, 1 = identical
|
|
*/
|
|
fuzzyMatch(target: string, ...searchTerms: string[]) {
|
|
if(searchTerms.length < 2) throw new Error('Requires at least 2 strings to compare');
|
|
const vector = (text: string, dimensions: number = 10): number[] => {
|
|
return text.toLowerCase().split('').map((char, index) =>
|
|
(char.charCodeAt(0) * (index + 1)) % dimensions / dimensions).slice(0, dimensions);
|
|
}
|
|
const v = vector(target);
|
|
const similarities = searchTerms.map(t => vector(t)).map(refVector => this.cosineSimilarity(v, refVector));
|
|
return {avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length, max: Math.max(...similarities), similarities};
|
|
}
|
|
|
|
/**
|
|
* Create a summary of some text
|
|
* @param {string} text Text to summarize
|
|
* @param {number} length Max number of words
|
|
* @param options LLM request options
|
|
* @returns {Promise<string>} Summary
|
|
*/
|
|
async summarize(text: string, length: number = 500, options?: LLMRequest): Promise<string | null> {
|
|
let system = `Your job is to summarize the users message using tool calls. Call the \`submit\` tool at least once with the shortest summary possible that's <= ${length} words. The tool call will respond with the token count. Responses are ignored`;
|
|
if(options?.system) system += '\n\n' + options.system;
|
|
return new Promise(async (resolve, reject) => {
|
|
let done = false;
|
|
const resp = await this.ask(text, {
|
|
temperature: 0.3,
|
|
...options,
|
|
system,
|
|
tools: [{
|
|
name: 'submit',
|
|
description: 'Submit summary',
|
|
args: {summary: {type: 'string', description: 'Text summarization', required: true}},
|
|
fn: (args) => {
|
|
if(!args.summary) return 'No summary provided';
|
|
const count = args.summary.split(' ').length;
|
|
if(count > length) return `Too long: ${length} words`;
|
|
done = true;
|
|
resolve(args.summary || null);
|
|
return `Saved: ${length} words`;
|
|
}
|
|
}, ...(options?.tools || [])],
|
|
});
|
|
if(!done) reject(`AI failed to create summary:\n${resp}`);
|
|
});
|
|
}
|
|
|
|
addModel(name: string, config: AnthropicConfig | OpenAiConfig, setDefault = false) {
|
|
if(config.proto == 'anthropic') this.models[name] = new Anthropic(this.ai, config.token, name);
|
|
else if(config.proto == 'openai') this.models[name] = new OpenAi(this.ai, config.host || null, config.token, name);
|
|
if(setDefault || !this.defaultModel) this.defaultModel = name;
|
|
}
|
|
|
|
removeModel(name: string) {
|
|
delete this.models[name];
|
|
if(this.defaultModel === name) {
|
|
this.defaultModel = Object.keys(this.models)[0] ?? '';
|
|
}
|
|
}
|
|
|
|
setModels(models: {[model: string]: AnthropicConfig | OpenAiConfig}, replace = true) {
|
|
if(replace) this.models = {};
|
|
Object.entries(models).forEach(([model, config]) => {
|
|
if(!this.defaultModel) this.defaultModel = model;
|
|
if(config.proto == 'anthropic') this.models[model] = new Anthropic(this.ai, config.token, model);
|
|
else if(config.proto == 'openai') this.models[model] = new OpenAi(this.ai, config.host || null, config.token, model);
|
|
});
|
|
this.defaultModel = Object.keys(this.models)[0] ?? '';
|
|
}
|
|
}
|
|
|
|
export default LLM;
|