263 lines
9.1 KiB
TypeScript
263 lines
9.1 KiB
TypeScript
import {JSONAttemptParse} from '@ztimson/utils';
|
|
import {AbortablePromise, Ai} from './ai.ts';
|
|
import {Anthropic} from './antrhopic.ts';
|
|
import {Ollama} from './ollama.ts';
|
|
import {OpenAi} from './open-ai.ts';
|
|
import {LLMProvider} from './provider.ts';
|
|
import {AiTool} from './tools.ts';
|
|
import {Worker} from 'worker_threads';
|
|
import {fileURLToPath} from 'url';
|
|
import {dirname, join} from 'path';
|
|
|
|
export type LLMMessage = {
|
|
/** Message originator */
|
|
role: 'assistant' | 'system' | 'user';
|
|
/** Message content */
|
|
content: string | any;
|
|
/** Timestamp */
|
|
timestamp?: number;
|
|
} | {
|
|
/** Tool call */
|
|
role: 'tool';
|
|
/** Unique ID for call */
|
|
id: string;
|
|
/** Tool that was run */
|
|
name: string;
|
|
/** Tool arguments */
|
|
args: any;
|
|
/** Tool result */
|
|
content: undefined | string;
|
|
/** Tool error */
|
|
error: undefined | string;
|
|
/** Timestamp */
|
|
timestamp?: number;
|
|
}
|
|
|
|
export type LLMOptions = {
|
|
/** Anthropic settings */
|
|
anthropic?: {
|
|
/** API Token */
|
|
token: string;
|
|
/** Default model */
|
|
model: string;
|
|
},
|
|
/** Ollama settings */
|
|
ollama?: {
|
|
/** connection URL */
|
|
host: string;
|
|
/** Default model */
|
|
model: string;
|
|
},
|
|
/** Open AI settings */
|
|
openAi?: {
|
|
/** API Token */
|
|
token: string;
|
|
/** Default model */
|
|
model: string;
|
|
},
|
|
/** Default provider & model */
|
|
model: string | [string, string];
|
|
} & Omit<LLMRequest, 'model'>;
|
|
|
|
export type LLMRequest = {
|
|
/** System prompt */
|
|
system?: string;
|
|
/** Message history */
|
|
history?: LLMMessage[];
|
|
/** Max tokens for request */
|
|
max_tokens?: number;
|
|
/** 0 = Rigid Logic, 1 = Balanced, 2 = Hyper Creative **/
|
|
temperature?: number;
|
|
/** Available tools */
|
|
tools?: AiTool[];
|
|
/** LLM model */
|
|
model?: string | [string, string];
|
|
/** Stream response */
|
|
stream?: (chunk: {text?: string, tool?: string, done?: true}) => any;
|
|
/** Compress old messages in the chat to free up context */
|
|
compress?: {
|
|
/** Trigger chat compression once context exceeds the token count */
|
|
max: number;
|
|
/** Compress chat until context size smaller than */
|
|
min: number
|
|
}
|
|
}
|
|
|
|
export class LLM {
|
|
private embedWorker: Worker | null = null;
|
|
private embedQueue = new Map<number, { resolve: (value: number[]) => void; reject: (error: any) => void }>();
|
|
private embedId = 0;
|
|
private providers: {[key: string]: LLMProvider} = {};
|
|
|
|
|
|
constructor(public readonly ai: Ai) {
|
|
this.embedWorker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'embedder.js'));
|
|
this.embedWorker.on('message', ({ id, embedding }) => {
|
|
const pending = this.embedQueue.get(id);
|
|
if (pending) {
|
|
pending.resolve(embedding);
|
|
this.embedQueue.delete(id);
|
|
}
|
|
});
|
|
|
|
if(ai.options.anthropic?.token) this.providers.anthropic = new Anthropic(this.ai, ai.options.anthropic.token, ai.options.anthropic.model);
|
|
if(ai.options.ollama?.host) this.providers.ollama = new Ollama(this.ai, ai.options.ollama.host, ai.options.ollama.model);
|
|
if(ai.options.openAi?.token) this.providers.openAi = new OpenAi(this.ai, ai.options.openAi.token, ai.options.openAi.model);
|
|
}
|
|
|
|
/**
|
|
* Chat with LLM
|
|
* @param {string} message Question
|
|
* @param {LLMRequest} options Configuration options and chat history
|
|
* @returns {{abort: () => void, response: Promise<LLMMessage[]>}} Function to abort response and chat history
|
|
*/
|
|
ask(message: string, options: LLMRequest = {}): AbortablePromise<LLMMessage[]> {
|
|
let model: any = [null, null];
|
|
if(options.model) {
|
|
if(typeof options.model == 'object') model = options.model;
|
|
else model = [options.model, (<any>this.ai.options)[options.model]?.model];
|
|
}
|
|
if(!options.model || model[1] == null) {
|
|
if(typeof this.ai.options.model == 'object') model = this.ai.options.model;
|
|
else model = [this.ai.options.model, (<any>this.ai.options)[this.ai.options.model]?.model];
|
|
}
|
|
if(!model[0] || !model[1]) throw new Error(`Unknown LLM provider or model: ${model[0]} / ${model[1]}`);
|
|
return this.providers[model[0]].ask(message, {...options, model: model[1]});
|
|
}
|
|
|
|
/**
|
|
* Compress chat history to reduce context size
|
|
* @param {LLMMessage[]} history Chatlog that will be compressed
|
|
* @param max Trigger compression once context is larger than max
|
|
* @param min Summarize until context size is less than min
|
|
* @param {LLMRequest} options LLM options
|
|
* @returns {Promise<LLMMessage[]>} New chat history will summary at index 0
|
|
*/
|
|
async compressHistory(history: LLMMessage[], max: number, min: number, options?: LLMRequest): Promise<LLMMessage[]> {
|
|
if(this.estimateTokens(history) < max) return history;
|
|
let keep = 0, tokens = 0;
|
|
for(let m of history.toReversed()) {
|
|
tokens += this.estimateTokens(m.content);
|
|
if(tokens < min) keep++;
|
|
else break;
|
|
}
|
|
if(history.length <= keep) return history;
|
|
const recent = keep == 0 ? [] : history.slice(-keep),
|
|
process = (keep == 0 ? history : history.slice(0, -keep)).filter(h => h.role === 'assistant' || h.role === 'user');
|
|
const summary = await this.summarize(process.map(m => `${m.role}: ${m.content}`).join('\n\n'), 250, options);
|
|
return [{role: 'assistant', content: `Conversation Summary: ${summary}`, timestamp: Date.now()}, ...recent];
|
|
}
|
|
|
|
cosineSimilarity(v1: number[], v2: number[]): number {
|
|
if (v1.length !== v2.length) throw new Error('Vectors must be same length');
|
|
let dotProduct = 0, normA = 0, normB = 0;
|
|
for (let i = 0; i < v1.length; i++) {
|
|
dotProduct += v1[i] * v2[i];
|
|
normA += v1[i] * v1[i];
|
|
normB += v2[i] * v2[i];
|
|
}
|
|
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
|
return denominator === 0 ? 0 : dotProduct / denominator;
|
|
}
|
|
|
|
chunk(target: object | string, maxTokens = 500, overlapTokens = 50): string[] {
|
|
const objString = (obj: any, path = ''): string[] => {
|
|
if(!obj) return [];
|
|
return Object.entries(obj).flatMap(([key, value]) => {
|
|
const p = path ? `${path}${isNaN(+key) ? `.${key}` : `[${key}]`}` : key;
|
|
if(typeof value === 'object' && !Array.isArray(value)) return objString(value, p);
|
|
return `${p}: ${Array.isArray(value) ? value.join(', ') : value}`;
|
|
});
|
|
};
|
|
|
|
const lines = typeof target === 'object' ? objString(target) : target.split('\n');
|
|
const tokens = lines.flatMap(l => [...l.split(/\s+/).filter(Boolean), '\n']);
|
|
const chunks: string[] = [];
|
|
for(let i = 0; i < tokens.length;) {
|
|
let text = '', j = i;
|
|
while(j < tokens.length) {
|
|
const next = text + (text ? ' ' : '') + tokens[j];
|
|
if(this.estimateTokens(next.replace(/\s*\n\s*/g, '\n')) > maxTokens && text) break;
|
|
text = next;
|
|
j++;
|
|
}
|
|
const clean = text.replace(/\s*\n\s*/g, '\n').trim();
|
|
if(clean) chunks.push(clean);
|
|
i = Math.max(j - overlapTokens, j === i ? i + 1 : j);
|
|
}
|
|
return chunks;
|
|
}
|
|
|
|
embedding(target: object | string, maxTokens = 500, overlapTokens = 50) {
|
|
const embed = (text: string): Promise<number[]> => {
|
|
return new Promise((resolve, reject) => {
|
|
const id = this.embedId++;
|
|
this.embedQueue.set(id, { resolve, reject });
|
|
this.embedWorker?.postMessage({ id, text });
|
|
});
|
|
};
|
|
|
|
const chunks = this.chunk(target, maxTokens, overlapTokens);
|
|
return Promise.all(chunks.map(async (text, index) => ({
|
|
index,
|
|
embedding: await embed(text),
|
|
text,
|
|
tokens: this.estimateTokens(text),
|
|
})));
|
|
}
|
|
|
|
/**
|
|
* Estimate variable as tokens
|
|
* @param history Object to size
|
|
* @returns {number} Rough token count
|
|
*/
|
|
estimateTokens(history: any): number {
|
|
const text = JSON.stringify(history);
|
|
return Math.ceil((text.length / 4) * 1.2);
|
|
}
|
|
|
|
/**
|
|
* Compare the difference between two strings using tensor math
|
|
* @param target Text that will checked
|
|
* @param {string} searchTerms Multiple search terms to check against target
|
|
* @returns {{avg: number, max: number, similarities: number[]}} Similarity values 0-1: 0 = unique, 1 = identical
|
|
*/
|
|
fuzzyMatch(target: string, ...searchTerms: string[]) {
|
|
if(searchTerms.length < 2) throw new Error('Requires at least 2 strings to compare');
|
|
const vector = (text: string, dimensions: number = 10): number[] => {
|
|
return text.toLowerCase().split('').map((char, index) =>
|
|
(char.charCodeAt(0) * (index + 1)) % dimensions / dimensions).slice(0, dimensions);
|
|
}
|
|
const v = vector(target);
|
|
const similarities = searchTerms.map(t => vector(t)).map(refVector => this.cosineSimilarity(v, refVector))
|
|
return {avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length, max: Math.max(...similarities), similarities}
|
|
}
|
|
|
|
/**
|
|
* Ask a question with JSON response
|
|
* @param {string} message Question
|
|
* @param {LLMRequest} options Configuration options and chat history
|
|
* @returns {Promise<{} | {} | RegExpExecArray | null>}
|
|
*/
|
|
async json(message: string, options?: LLMRequest) {
|
|
let resp = await this.ask(message, {
|
|
system: 'Respond using a JSON blob',
|
|
...options
|
|
});
|
|
if(!resp?.[0]?.content) return {};
|
|
return JSONAttemptParse(new RegExp('\{[\s\S]*\}').exec(resp[0].content), {});
|
|
}
|
|
|
|
/**
|
|
* Create a summary of some text
|
|
* @param {string} text Text to summarize
|
|
* @param {number} tokens Max number of tokens
|
|
* @param options LLM request options
|
|
* @returns {Promise<string>} Summary
|
|
*/
|
|
summarize(text: string, tokens: number, options?: LLMRequest): Promise<string | null> {
|
|
return this.ask(text, {system: `Generate a brief summary <= ${tokens} tokens. Output nothing else`, temperature: 0.3, ...options})
|
|
.then(history => <string>history.pop()?.content || null);
|
|
}
|
|
}
|