Compare commits
61 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d53b1c6328 | |||
| 89619e211e | |||
| afc6653364 | |||
| 68e72445a2 | |||
| 1aa6cdf329 | |||
| d022a5ef4d | |||
| a1d438a20a | |||
| 52a9e3aaa4 | |||
| a7aec4ee29 | |||
| dda2d4c2a3 | |||
| 58e0e488e4 | |||
| 8dfcd06752 | |||
| 14f6cdd313 | |||
| 73d6ee0f2a | |||
| bee4085666 | |||
| 3b5c71de7c | |||
| 8229e02a52 | |||
| a6fb8ae828 | |||
| d1230bcaad | |||
| 2d49c9aa80 | |||
| 9a39f00f94 | |||
| 436757daad | |||
| 69b3297bb3 | |||
| 710c6ce52c | |||
| 4ac3036000 | |||
| 3121d542d4 | |||
| 51ab8f2538 | |||
| 7dd3307a07 | |||
| 209d3b120b | |||
| 0b1c25dfda | |||
| af6522ad88 | |||
| ee7b85301b | |||
| d2e711fbf2 | |||
| 596e99daa7 | |||
| eda4eed87d | |||
| 7f88c2d1d0 | |||
| 5eae84f6cf | |||
| 52a3e73484 | |||
| ccb1bdf043 | |||
| b814ea8b28 | |||
| 06dda88dbc | |||
| 5d34652d46 | |||
| 6454548364 | |||
| 936317f2f2 | |||
| cfde2ac4d3 | |||
| e4ba89d3db | |||
| 71a7e2a904 | |||
| abd290246c | |||
| ca66e8e304 | |||
| cec892563e | |||
| 91066e070f | |||
| a94b153c6d | |||
| 39537a4a8f | |||
| 790608f020 | |||
| 473424ae23 | |||
| 9b831f7d95 | |||
| 498b326e45 | |||
| 56e4efec94 | |||
| a07f069ad0 | |||
| da15d299e6 | |||
| 7ef7c3f676 |
138
README.md
138
README.md
@@ -3,7 +3,7 @@
|
||||
<br />
|
||||
|
||||
<!-- Logo -->
|
||||
<img src="https://git.zakscode.com/repo-avatars/a90851ca730480ec37a5c0c2c4f1b4609eee5eadf806eaf16c83ac4cb7493aa9" alt="Logo" width="200" height="200">
|
||||
<img alt="Logo" width="200" height="200" src="https://git.zakscode.com/repo-avatars/a82d423674763e7a0c1c945bdbb07e249b2bb786d3c9beae76d5b196a10f5c0f">
|
||||
|
||||
<!-- Title -->
|
||||
### @ztimson/ai-utils
|
||||
@@ -53,13 +53,15 @@ A TypeScript library that provides a unified interface for working with multiple
|
||||
- **Provider Abstraction**: Switch between AI providers without changing your code
|
||||
|
||||
### Built With
|
||||
[](https://anthropic.com/)
|
||||
[](https://openai.com/)
|
||||
[](https://ollama.com/)
|
||||
[](https://tensorflow.org/)
|
||||
[](https://tesseract-ocr.github.io/)
|
||||
[](https://anthropic.com/)
|
||||
[](https://github.com/ggml-org/llama.cpp)
|
||||
[](https://openai.com/)
|
||||
[](https://github.com/pyannote)
|
||||
[](https://tensorflow.org/)
|
||||
[](https://tesseract-ocr.github.io/)
|
||||
[](https://huggingface.co/docs/transformers.js/en/index)
|
||||
[](https://typescriptlang.org/)
|
||||
[](https://github.com/ggerganov/whisper.cpp)
|
||||
[](https://github.com/ggerganov/whisper.cpp)
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -88,6 +90,8 @@ A TypeScript library that provides a unified interface for working with multiple
|
||||
|
||||
#### Prerequisites
|
||||
- [Node.js](https://nodejs.org/en/download)
|
||||
- _[Whisper.cpp](https://github.com/ggml-org/whisper.cpp/releases/tag) (ASR)_
|
||||
- _[Pyannote](https://github.com/pyannote) (ASR Diarization):_ `pip install pyannote.audio`
|
||||
|
||||
#### Instructions
|
||||
1. Install the dependencies: `npm i`
|
||||
@@ -99,7 +103,125 @@ A TypeScript library that provides a unified interface for working with multiple
|
||||
|
||||
## Documentation
|
||||
|
||||
[Available Here](https://ai-utils.docs.zakscode.com/)
|
||||
### Setup
|
||||
```javascript
|
||||
const ai = new Ai({
|
||||
path: '/ai-models',
|
||||
|
||||
// Setup audio
|
||||
whisper: '/path/to/binary', // Required for ASR
|
||||
hfToken: '...', // Required for diarization
|
||||
asr: 'ggml-base.en.bin', // Override default ASR model
|
||||
|
||||
// Setup LLM
|
||||
embedder: 'bge-small-en-v1.5', // Override default embedder model
|
||||
llm: {
|
||||
system: 'You are a helpful assistant.',
|
||||
compress: {max: 90_000, min: 50_000}, // Compress chat history to min tokens when max is reached
|
||||
temperature: 0.8,
|
||||
max_tokens: 100_000,
|
||||
memoryModel: 'gpt-4o', // Cheap model for managing memories in background, defaults to current model
|
||||
models: {
|
||||
'claude-3-5-sonnet': {proto: 'anthropic', token: process.env.ANTHROPIC_TOKEN},
|
||||
'gpt-4o': {proto: 'openai', token: process.env.OPENAI_TOKEN},
|
||||
'llama3': {proto: 'ollama', host: 'http://localhost:11434'},
|
||||
},
|
||||
mcp: [
|
||||
{name: 'files', url: 'https://mcp.example.com', token: process.env.MCP_TOKEN}
|
||||
],
|
||||
skills: [
|
||||
{name: 'Tone of voice', description: 'Brand writing guidelines', content: '# Tone of Voice\n\nAlways be concise and friendly...'}
|
||||
],
|
||||
tools: [{
|
||||
name: 'Marco?',
|
||||
description: 'Where is marco polo?',
|
||||
args: {
|
||||
shout: {type: 'boolean', default: 'Shout into the void?', description: false, required: false}
|
||||
},
|
||||
fn: (args: any, stream: LLMRequest['stream'], ai: Ai) => {
|
||||
const {shout} = args;
|
||||
return shout ? 'Polo!' : 'Polo';
|
||||
}
|
||||
}],
|
||||
},
|
||||
|
||||
// Setup Vision
|
||||
ocr: 'eng' // Override default OCR model
|
||||
});
|
||||
|
||||
```
|
||||
|
||||
### Audio
|
||||
|
||||
```javascript
|
||||
// Crate audio transcript
|
||||
const text = await ai.audio.asr('./path/to/audio.mp3');
|
||||
console.log(text);
|
||||
|
||||
// Break transcript into speakers
|
||||
const text = await ai.audio.asr('./path/to/audio.mp3', {diarization: true});
|
||||
console.log(text);
|
||||
|
||||
// Break transcript into named speakers
|
||||
const text = await ai.audio.asr('./path/to/audio.mp3', {diarization: 'llm'});
|
||||
console.log(text);
|
||||
```
|
||||
|
||||
### Language
|
||||
|
||||
```javascript
|
||||
const history = [], memory = [];
|
||||
|
||||
// Wait for entire response
|
||||
const text = await ai.language.ask('My favorite color is blue, whats yours?', {history, memory});
|
||||
console.log(text);
|
||||
|
||||
// Stream response
|
||||
const chunks = '';
|
||||
await ai.language.ask('Write me a poem', {
|
||||
history, memory,
|
||||
stream: chunk => chunks += chunk,
|
||||
});
|
||||
console.log(chunks);
|
||||
|
||||
// Manually compile history into memories at end of conversation
|
||||
// Happens automatically when coverstaions are compressed
|
||||
await ai.language.updateMemory(history, memory);
|
||||
|
||||
// Summarize text
|
||||
const summary = await ai.language.summarize(longText, 200);
|
||||
|
||||
// Code response (no conversation or extra BS)
|
||||
const code = await ai.language.code('Write a fibonacci function');
|
||||
|
||||
// Structured JSON response
|
||||
const data = await ai.language.json('Extract the name and age', `{
|
||||
"name": "string",
|
||||
"age": "number"
|
||||
}`, {system: 'Extract from user input'});
|
||||
```
|
||||
|
||||
#### Premade LLM Tools:
|
||||
- `cli`: Run a shell command, returns its output
|
||||
- `get_datetime`: Returns local date/time
|
||||
- `get_datetime_utc`: Returns current UTC date/time
|
||||
- `exec`: Execute code in cli, node, or python
|
||||
- `fetch`: Make HTTP requests (GET/POST/PUT/DELETE)
|
||||
- `exec_javascript`: Execute CommonJS JavaScript
|
||||
- `exec_python`: Execute Python via python -c
|
||||
- `read_webpage`: Scrape & clean content from a URL, handles HTML, JSON, CSV, media, PDFs etc.
|
||||
- `web_search`: Anonymous DuckDuckGo search, returns a list of URLs
|
||||
- `wikipedia_lookup`: Fetch a Wikipedia article (intro or full)
|
||||
- `wikipedia_search`: Search Wikipedia and return matching articles
|
||||
- `get_weather`: Fetch current weather + forecast for a location (just built!)
|
||||
|
||||
### Vision
|
||||
|
||||
```javascript
|
||||
// Extract text from image
|
||||
const text = await ai.vision.ocr('./path/to/image.png');
|
||||
console.log(text);
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
|
||||
4149
package-lock.json
generated
4149
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
23
package.json
23
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@ztimson/ai-utils",
|
||||
"version": "0.7.0",
|
||||
"version": "1.3.4",
|
||||
"description": "AI Utility library",
|
||||
"author": "Zak Timson",
|
||||
"license": "MIT",
|
||||
@@ -25,22 +25,21 @@
|
||||
"watch": "npx vite build --watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.67.0",
|
||||
"@anthropic-ai/sdk": "^0.102.0",
|
||||
"@tensorflow/tfjs": "^4.22.0",
|
||||
"@xenova/transformers": "^2.17.2",
|
||||
"@ztimson/node-utils": "^1.0.4",
|
||||
"@ztimson/utils": "^0.27.9",
|
||||
"@huggingface/transformers": "^4.2.0",
|
||||
"@ztimson/node-utils": "^1.0.7",
|
||||
"@ztimson/utils": "^0.29.4",
|
||||
"cheerio": "^1.2.0",
|
||||
"openai": "^6.6.0",
|
||||
"tesseract.js": "^6.0.1",
|
||||
"wavefile": "^11.0.0"
|
||||
"openai": "^6.42.0",
|
||||
"tesseract.js": "^7.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"@types/node": "^24.13.1",
|
||||
"typedoc": "^0.26.7",
|
||||
"typescript": "^5.3.3",
|
||||
"vite": "^7.2.7",
|
||||
"vite-plugin-dts": "^4.5.3"
|
||||
"typescript": "^5.6.3",
|
||||
"vite": "^8.0.16",
|
||||
"vite-plugin-dts": "^5.0.2"
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
|
||||
10
src/ai.ts
10
src/ai.ts
@@ -1,5 +1,5 @@
|
||||
import * as os from 'node:os';
|
||||
import LLM, {AnthropicConfig, OllamaConfig, OpenAiConfig, LLMRequest} from './llm';
|
||||
import LLM, {AnthropicConfig, OpenAiConfig, LLMRequest} from './llm';
|
||||
import { Audio } from './audio.ts';
|
||||
import {Vision} from './vision.ts';
|
||||
|
||||
@@ -8,20 +8,22 @@ export type AbortablePromise<T> = Promise<T> & {
|
||||
};
|
||||
|
||||
export type AiOptions = {
|
||||
/** Token to pull models from hugging face */
|
||||
/** Token to pull diarization models from hugging face */
|
||||
hfToken?: string;
|
||||
/** Path to models */
|
||||
path?: string;
|
||||
/** ASR model: whisper-tiny, whisper-base */
|
||||
/** Whisper ASR model: ggml-tiny.en.bin, ggml-base.en.bin */
|
||||
asr?: string;
|
||||
/** Embedding model: all-MiniLM-L6-v2, bge-small-en-v1.5, bge-large-en-v1.5 */
|
||||
embedder?: string;
|
||||
/** Large language models, first is default */
|
||||
llm?: Omit<LLMRequest, 'model'> & {
|
||||
models: {[model: string]: AnthropicConfig | OllamaConfig | OpenAiConfig};
|
||||
models: {[model: string]: AnthropicConfig | OpenAiConfig};
|
||||
}
|
||||
/** OCR model: eng, eng_best, eng_fast */
|
||||
ocr?: string;
|
||||
/** Whisper binary */
|
||||
whisper?: string;
|
||||
}
|
||||
|
||||
export class Ai {
|
||||
|
||||
@@ -3,6 +3,7 @@ import {findByProp, objectMap, JSONSanitize, JSONAttemptParse} from '@ztimson/ut
|
||||
import {AbortablePromise, Ai} from './ai.ts';
|
||||
import {LLMMessage, LLMRequest} from './llm.ts';
|
||||
import {LLMProvider} from './provider.ts';
|
||||
import {convertSchema} from './tools.ts';
|
||||
|
||||
export class Anthropic extends LLMProvider {
|
||||
client!: anthropic;
|
||||
@@ -20,10 +21,10 @@ export class Anthropic extends LLMProvider {
|
||||
messages.push(<any>{timestamp, ...h});
|
||||
} else {
|
||||
const textContent = h.content?.filter((c: any) => c.type == 'text').map((c: any) => c.text).join('\n\n');
|
||||
if(textContent) messages.push({timestamp, role: h.role, content: textContent});
|
||||
if(textContent) messages.push({role: h.role, content: textContent, timestamp: timestamp});
|
||||
h.content.forEach((c: any) => {
|
||||
if(c.type == 'tool_use') {
|
||||
messages.push({timestamp, role: 'tool', id: c.id, name: c.name, args: c.input, content: undefined});
|
||||
messages.push({role: 'tool', id: c.id, name: c.name, args: c.input, timestamp: c.timestamp, content: undefined});
|
||||
} else if(c.type == 'tool_result') {
|
||||
const m: any = messages.findLast(m => (<any>m).id == c.tool_use_id);
|
||||
if(m) m[c.is_error ? 'error' : 'content'] = c.content;
|
||||
@@ -45,19 +46,22 @@ export class Anthropic extends LLMProvider {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return history.map(({timestamp, ...h}) => h);
|
||||
return history;
|
||||
}
|
||||
|
||||
ask(message: string, options: LLMRequest = {}): AbortablePromise<string> {
|
||||
ask(message: string, options: LLMRequest = {}): AbortablePromise<string | any> {
|
||||
const controller = new AbortController();
|
||||
return Object.assign(new Promise<any>(async (res) => {
|
||||
let history = this.fromStandard([...options.history || [], {role: 'user', content: message, timestamp: Date.now()}]);
|
||||
let history = this.fromStandard([
|
||||
...(options.history || []).filter(h => h.role !== 'system'),
|
||||
{role: 'user', content: message, timestamp: Date.now()}
|
||||
]);
|
||||
const tools = options.tools || this.ai.options.llm?.tools || [];
|
||||
const requestParams: any = {
|
||||
model: options.model || this.model,
|
||||
max_tokens: options.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
||||
system: options.system || this.ai.options.llm?.system || '',
|
||||
temperature: options.temperature || this.ai.options.llm?.temperature || 0.7,
|
||||
temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
|
||||
tools: tools.map(t => ({
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
@@ -72,8 +76,19 @@ export class Anthropic extends LLMProvider {
|
||||
stream: !!options.stream,
|
||||
};
|
||||
|
||||
let resp: any, isFirstMessage = true;
|
||||
// Add structured output support
|
||||
if(options.schema) {
|
||||
requestParams.output_config = {
|
||||
format: {
|
||||
type: 'json_schema',
|
||||
schema: convertSchema(options.schema)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let resp: any, terminal = false;
|
||||
do {
|
||||
requestParams.messages = history.map(({timestamp, ...m}) => m);
|
||||
resp = await this.client.messages.create(requestParams).catch(err => {
|
||||
err.message += `\n\nMessages:\n${JSON.stringify(history, null, 2)}`;
|
||||
throw err;
|
||||
@@ -81,8 +96,6 @@ export class Anthropic extends LLMProvider {
|
||||
|
||||
// Streaming mode
|
||||
if(options.stream) {
|
||||
if(!isFirstMessage) options.stream({text: '\n\n'});
|
||||
else isFirstMessage = false;
|
||||
resp.content = [];
|
||||
for await (const chunk of resp) {
|
||||
if(controller.signal.aborted) break;
|
||||
@@ -102,7 +115,7 @@ export class Anthropic extends LLMProvider {
|
||||
}
|
||||
} else if(chunk.type === 'content_block_stop') {
|
||||
const last = resp.content.at(-1);
|
||||
if(last.input != null) last.input = last.input ? JSONAttemptParse(last.input, {}) : {};
|
||||
if(last?.input != null) last.input = last.input ? JSONAttemptParse(last.input, {}) : {};
|
||||
} else if(chunk.type === 'message_stop') {
|
||||
break;
|
||||
}
|
||||
@@ -112,28 +125,43 @@ export class Anthropic extends LLMProvider {
|
||||
// Run tools
|
||||
const toolCalls = resp.content.filter((c: any) => c.type === 'tool_use');
|
||||
if(toolCalls.length && !controller.signal.aborted) {
|
||||
history.push({role: 'assistant', content: resp.content});
|
||||
history.push({role: 'assistant', content: resp.content, timestamp: Date.now()});
|
||||
const results = await Promise.all(toolCalls.map(async (toolCall: any) => {
|
||||
const tool = tools.find(findByProp('name', toolCall.name));
|
||||
if(options.stream) options.stream({tool: toolCall.name});
|
||||
if(!tool) return {tool_use_id: toolCall.id, is_error: true, content: 'Tool not found'};
|
||||
try {
|
||||
const result = await tool.fn(toolCall.input, options?.stream, this.ai);
|
||||
return {type: 'tool_result', tool_use_id: toolCall.id, content: JSONSanitize(result)};
|
||||
// Wrap stream so a tool's `done` ends turn gracefully
|
||||
const toolStream = options.stream && ((chunk: any) => {
|
||||
if(chunk.done) { terminal = true; return; }
|
||||
options.stream!(chunk);
|
||||
});
|
||||
const result = await tool.fn(toolCall.input, toolStream, this.ai, toolCall.id);
|
||||
return {type: 'tool_result', tool_use_id: toolCall.id, content: typeof result == 'object' ? JSONSanitize(result) : result};
|
||||
} catch (err: any) {
|
||||
return {type: 'tool_result', tool_use_id: toolCall.id, is_error: true, content: err?.message || err?.toString() || 'Unknown'};
|
||||
}
|
||||
}));
|
||||
history.push({role: 'user', content: results});
|
||||
history.push({role: 'user', content: results, timestamp: Date.now()});
|
||||
requestParams.messages = history;
|
||||
}
|
||||
} while (!controller.signal.aborted && resp.content.some((c: any) => c.type === 'tool_use'));
|
||||
history.push({role: 'assistant', content: resp.content.filter((c: any) => c.type == 'text').map((c: any) => c.text).join('\n\n')});
|
||||
history = this.toStandard(history);
|
||||
} while (!terminal && !controller.signal.aborted && resp.content.some((c: any) => c.type === 'tool_use'));
|
||||
|
||||
if(options.stream) options.stream({done: true});
|
||||
if(!terminal) {
|
||||
const textContent = resp.content.filter((c: any) => c.type == 'text').map((c: any) => c.text).join('\n\n');
|
||||
history.push({role: 'assistant', content: textContent.trim(), timestamp: Date.now()});
|
||||
}
|
||||
history = this.toStandard(history);
|
||||
if(options.history) options.history.splice(0, options.history.length, ...history);
|
||||
res(history.at(-1)?.content);
|
||||
if(options.stream) options.stream({done: true});
|
||||
|
||||
const turnStart = history.map(h => h.role).lastIndexOf('user');
|
||||
const finalContent = history.slice(turnStart + 1).reduce((str, h) => {
|
||||
if(h.role === 'assistant') return str + (h.content || '');
|
||||
return str;
|
||||
}, '').trim();
|
||||
|
||||
res(options.schema ? JSONAttemptParse(finalContent, finalContent) : finalContent);
|
||||
}), {abort: () => controller.abort()});
|
||||
}
|
||||
}
|
||||
|
||||
134
src/asr.ts
134
src/asr.ts
@@ -1,134 +0,0 @@
|
||||
import { pipeline } from '@xenova/transformers';
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { mkdtempSync, rmSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import wavefile from 'wavefile';
|
||||
|
||||
let whisperPipeline: any;
|
||||
|
||||
export async function canDiarization(): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const proc = spawn('python', ['-c', 'import pyannote.audio']);
|
||||
proc.on('close', (code: number) => resolve(code === 0));
|
||||
proc.on('error', () => resolve(false));
|
||||
});
|
||||
}
|
||||
|
||||
async function runDiarization(audioPath: string, dir: string, token: string): Promise<any[]> {
|
||||
const script = `
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from pyannote.audio import Pipeline
|
||||
|
||||
os.environ['TORCH_HOME'] = r"${dir}"
|
||||
pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token="${token}")
|
||||
output = pipeline(sys.argv[1])
|
||||
|
||||
segments = []
|
||||
for turn, speaker in output.speaker_diarization:
|
||||
segments.append({"start": turn.start, "end": turn.end, "speaker": speaker})
|
||||
|
||||
print(json.dumps(segments))
|
||||
`;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let output = '';
|
||||
const proc = spawn('python', ['-c', script, audioPath]);
|
||||
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
||||
proc.stderr.on('data', (data: Buffer) => console.error(data.toString()));
|
||||
proc.on('close', (code: number) => {
|
||||
if(code === 0) {
|
||||
try {
|
||||
resolve(JSON.parse(output));
|
||||
} catch (err) {
|
||||
reject(new Error('Failed to parse diarization output'));
|
||||
}
|
||||
} else {
|
||||
reject(new Error(`Python process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
proc.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function combineSpeakerTranscript(chunks: any[], speakers: any[]): string {
|
||||
const speakerMap = new Map();
|
||||
let speakerCount = 0;
|
||||
speakers.forEach((seg: any) => {
|
||||
if(!speakerMap.has(seg.speaker)) speakerMap.set(seg.speaker, ++speakerCount);
|
||||
});
|
||||
|
||||
const lines: string[] = [];
|
||||
let currentSpeaker = -1;
|
||||
let currentText = '';
|
||||
chunks.forEach((chunk: any) => {
|
||||
const time = chunk.timestamp[0];
|
||||
const speaker = speakers.find((s: any) => time >= s.start && time <= s.end);
|
||||
const speakerNum = speaker ? speakerMap.get(speaker.speaker) : 1;
|
||||
if (speakerNum !== currentSpeaker) {
|
||||
if(currentText) lines.push(`[Speaker ${currentSpeaker}]: ${currentText.trim()}`);
|
||||
currentSpeaker = speakerNum;
|
||||
currentText = chunk.text;
|
||||
} else {
|
||||
currentText += chunk.text;
|
||||
}
|
||||
});
|
||||
if(currentText) lines.push(`[Speaker ${currentSpeaker}]: ${currentText.trim()}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function prepareAudioBuffer(file: string): [string, Float32Array] {
|
||||
let wav: any, tmp;
|
||||
try {
|
||||
wav = new wavefile.WaveFile(readFileSync(file));
|
||||
} catch(err) {
|
||||
tmp = join(mkdtempSync(join(tmpdir(), 'audio-')), 'converted.wav');
|
||||
execSync(`ffmpeg -i "${file}" -ar 16000 -ac 1 -f wav "${tmp}"`, { stdio: 'ignore' });
|
||||
wav = new wavefile.WaveFile(readFileSync(tmp));
|
||||
} finally {
|
||||
wav.toBitDepth('32f');
|
||||
wav.toSampleRate(16000);
|
||||
const samples = wav.getSamples();
|
||||
if(Array.isArray(samples)) {
|
||||
const left = samples[0];
|
||||
const right = samples[1];
|
||||
const buffer = new Float32Array(left.length);
|
||||
for (let i = 0; i < left.length; i++) buffer[i] = (left[i] + right[i]) / 2;
|
||||
return [tmp || file, buffer];
|
||||
}
|
||||
return [tmp || file, samples];
|
||||
}
|
||||
}
|
||||
|
||||
parentPort?.on('message', async ({ file, speaker, model, modelDir, token }) => {
|
||||
try {
|
||||
if(!whisperPipeline) whisperPipeline = await pipeline('automatic-speech-recognition', `Xenova/${model}`, {cache_dir: modelDir, quantized: true});
|
||||
|
||||
// Prepare audio file
|
||||
const [f, buffer] = prepareAudioBuffer(file);
|
||||
|
||||
// Fetch transcript and speakers
|
||||
const hasDiarization = speaker && await canDiarization();
|
||||
const [transcript, speakers] = await Promise.all([
|
||||
whisperPipeline(buffer, {return_timestamps: speaker ? 'word' : false}),
|
||||
(!speaker || !token || !hasDiarization) ? Promise.resolve(): runDiarization(f, modelDir, token),
|
||||
]);
|
||||
if(file != f) rmSync(f, { recursive: true, force: true });
|
||||
|
||||
// Return any results / errors if no more processing required
|
||||
const text = transcript.text?.trim() || null;
|
||||
if(!speaker) return parentPort?.postMessage({ text });
|
||||
if(!token) return parentPort?.postMessage({ text, error: 'HuggingFace token required' });
|
||||
if(!hasDiarization) return parentPort?.postMessage({ text, error: 'Speaker diarization unavailable' });
|
||||
|
||||
// Combine transcript and speakers
|
||||
const combined = combineSpeakerTranscript(transcript.chunks || [], speakers || []);
|
||||
parentPort?.postMessage({ text: combined });
|
||||
} catch (err: any) {
|
||||
parentPort?.postMessage({ error: err.stack || err.message });
|
||||
}
|
||||
});
|
||||
300
src/audio.ts
300
src/audio.ts
@@ -1,58 +1,276 @@
|
||||
import {fileURLToPath} from 'url';
|
||||
import {Worker} from 'worker_threads';
|
||||
import {execSync, spawn} from 'node:child_process';
|
||||
import {mkdtempSync} from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import {tmpdir} from 'node:os';
|
||||
import Path, {join} from 'node:path';
|
||||
import {AbortablePromise, Ai} from './ai.ts';
|
||||
import {canDiarization} from './asr.ts';
|
||||
import {dirname, join} from 'path';
|
||||
|
||||
export class Audio {
|
||||
constructor(private ai: Ai) {}
|
||||
private downloads: {[key: string]: Promise<string>} = {};
|
||||
private pyannote!: string;
|
||||
private whisperModel!: string;
|
||||
|
||||
asr(file: string, options: { model?: string; speaker?: boolean | 'id' } = {}): AbortablePromise<string | null> {
|
||||
const { model = this.ai.options.asr || 'whisper-base', speaker = false } = options;
|
||||
let aborted = false;
|
||||
const abort = () => { aborted = true; };
|
||||
|
||||
let p = new Promise<string | null>((resolve, reject) => {
|
||||
const worker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'asr.js'));
|
||||
const handleMessage = ({ text, warning, error }: any) => {
|
||||
worker.terminate();
|
||||
if(aborted) return;
|
||||
if(error) reject(new Error(error));
|
||||
else {
|
||||
if(warning) console.warn(warning);
|
||||
resolve(text);
|
||||
constructor(private ai: Ai) {
|
||||
if(ai.options.whisper) {
|
||||
this.whisperModel = ai.options.asr || 'ggml-base.en.bin';
|
||||
this.downloadAsrModel();
|
||||
}
|
||||
|
||||
this.pyannote = `
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from pyannote.audio import Pipeline
|
||||
|
||||
os.environ['TORCH_HOME'] = r"${ai.options.path}"
|
||||
pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token="${ai.options.hfToken}")
|
||||
output = pipeline(sys.argv[1])
|
||||
|
||||
segments = []
|
||||
for turn, speaker in output.speaker_diarization:
|
||||
segments.append({"start": turn.start, "end": turn.end, "speaker": speaker})
|
||||
|
||||
print(json.dumps(segments))
|
||||
`;
|
||||
}
|
||||
|
||||
private async addPunctuation(timestampData: any, llm?: boolean, cadence = 150): Promise<string> {
|
||||
const countSyllables = (word: string): number => {
|
||||
word = word.toLowerCase().replace(/[^a-z]/g, '');
|
||||
if(word.length <= 3) return 1;
|
||||
const matches = word.match(/[aeiouy]+/g);
|
||||
let count = matches ? matches.length : 1;
|
||||
if(word.endsWith('e')) count--;
|
||||
return Math.max(1, count);
|
||||
};
|
||||
const handleError = (err: Error) => {
|
||||
worker.terminate();
|
||||
if(!aborted) reject(err);
|
||||
};
|
||||
worker.on('message', handleMessage);
|
||||
worker.on('error', handleError);
|
||||
worker.on('exit', (code) => {
|
||||
if(code !== 0 && !aborted) reject(new Error(`Worker exited with code ${code}`));
|
||||
|
||||
let result = '';
|
||||
timestampData.transcription.filter((word, i) => {
|
||||
let skip = false;
|
||||
const prevWord = timestampData.transcription[i - 1];
|
||||
const nextWord = timestampData.transcription[i + 1];
|
||||
if(!word.text && nextWord) {
|
||||
nextWord.offsets.from = word.offsets.from;
|
||||
nextWord.timestamps.from = word.offsets.from;
|
||||
} else if(word.text && word.text[0] != ' ' && prevWord) {
|
||||
prevWord.offsets.to = word.offsets.to;
|
||||
prevWord.timestamps.to = word.timestamps.to;
|
||||
prevWord.text += word.text;
|
||||
skip = true;
|
||||
}
|
||||
return !!word.text && !skip;
|
||||
}).forEach((word: any) => {
|
||||
const capital = /^[A-Z]/.test(word.text.trim());
|
||||
const length = word.offsets.to - word.offsets.from;
|
||||
const syllables = countSyllables(word.text.trim());
|
||||
const expected = syllables * cadence;
|
||||
if(capital && length > expected * 2 && word.text[0] == ' ') result += '.';
|
||||
result += word.text;
|
||||
});
|
||||
worker.postMessage({file, model, speaker, modelDir: this.ai.options.path, token: this.ai.options.hfToken});
|
||||
if(!llm) return result.trim();
|
||||
return this.ai.language.ask(result, {
|
||||
system: 'Remove any misplaced punctuation from the following ASR transcript using the replace tool. Avoid modifying words unless there is an obvious typo',
|
||||
temperature: 0.1,
|
||||
tools: [{
|
||||
name: 'replace',
|
||||
description: 'Use find and replace to fix errors',
|
||||
args: {
|
||||
find: {type: 'string', description: 'Text to find', required: true},
|
||||
replace: {type: 'string', description: 'Text to replace', required: true}
|
||||
},
|
||||
fn: (args) => result = result.replace(args.find, args.replace)
|
||||
}]
|
||||
}).then(() => result);
|
||||
}
|
||||
|
||||
private async diarizeTranscript(timestampData: any, speakers: any[], llm: boolean): Promise<string> {
|
||||
const speakerMap = new Map();
|
||||
let speakerCount = 0;
|
||||
speakers.forEach((seg: any) => {
|
||||
if(!speakerMap.has(seg.speaker)) speakerMap.set(seg.speaker, ++speakerCount);
|
||||
});
|
||||
|
||||
// Name speakers using AI
|
||||
if(options.speaker == 'id') {
|
||||
if(!this.ai.language.defaultModel) throw new Error('Configure an LLM for advanced ASR speaker detection');
|
||||
p = p.then(async transcript => {
|
||||
if(!transcript) return transcript;
|
||||
const names = await this.ai.language.json(transcript, '{1: "Detected Name"}', {
|
||||
system: 'Use this following transcript to identify speakers. Only identify speakers you are sure about',
|
||||
temperature: 0.2,
|
||||
const punctuatedText = await this.addPunctuation(timestampData, llm);
|
||||
const sentences = punctuatedText.match(/[^.!?]+[.!?]+/g) || [punctuatedText];
|
||||
const words = timestampData.transcription.filter((w: any) => w.text.trim());
|
||||
|
||||
// Assign speaker to each sentence
|
||||
const sentencesWithSpeakers = sentences.map(sentence => {
|
||||
sentence = sentence.trim();
|
||||
if(!sentence) return null;
|
||||
|
||||
const sentenceWords = sentence.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
|
||||
const speakerWordCount = new Map<number, number>();
|
||||
|
||||
sentenceWords.forEach(sw => {
|
||||
const word = words.find((w: any) => sw === w.text.trim().toLowerCase().replace(/[^\w]/g, ''));
|
||||
if(!word) return;
|
||||
|
||||
const wordTime = word.offsets.from / 1000;
|
||||
const speaker = speakers.find((seg: any) => wordTime >= seg.start && wordTime <= seg.end);
|
||||
if(speaker) {
|
||||
const spkNum = speakerMap.get(speaker.speaker);
|
||||
speakerWordCount.set(spkNum, (speakerWordCount.get(spkNum) || 0) + 1);
|
||||
}
|
||||
});
|
||||
Object.entries(names).forEach(([speaker, name]) => {
|
||||
transcript = (<string>transcript).replaceAll(`[Speaker ${speaker}]`, `[${name}]`);
|
||||
|
||||
let bestSpeaker = 1;
|
||||
let maxWords = 0;
|
||||
speakerWordCount.forEach((count, speaker) => {
|
||||
if(count > maxWords) {
|
||||
maxWords = count;
|
||||
bestSpeaker = speaker;
|
||||
}
|
||||
});
|
||||
|
||||
return {speaker: bestSpeaker, text: sentence};
|
||||
}).filter(s => s !== null);
|
||||
|
||||
// Merge adjacent sentences from same speaker
|
||||
const merged: Array<{speaker: number, text: string}> = [];
|
||||
sentencesWithSpeakers.forEach(item => {
|
||||
const last = merged[merged.length - 1];
|
||||
if(last && last.speaker === item.speaker) {
|
||||
last.text += ' ' + item.text;
|
||||
} else {
|
||||
merged.push({...item});
|
||||
}
|
||||
});
|
||||
|
||||
let transcript = merged.map(item => `[Speaker ${item.speaker}]: ${item.text}`).join('\n').trim();
|
||||
if(!llm) return transcript;
|
||||
let chunks = this.ai.language.chunk(transcript, 500, 0);
|
||||
if(chunks.length > 4) chunks = [...chunks.slice(0, 3), <string>chunks.at(-1)];
|
||||
await this.ai.language.ask(chunks.join('\n'), {
|
||||
system: 'Read the following transcript and attempt to identify every speaker. For every positively identified speaker, call the \`identify\` tool with the speaker\'s ID number & the identified name exactly once.',
|
||||
temperature: 0.1,
|
||||
tools: [
|
||||
{name: 'identify', description: 'Identify a speaker', args: {
|
||||
speaker: {type: 'number', description: 'Speaker number', required: true},
|
||||
name: {type: 'string', description: 'Inferred name', required: true},
|
||||
}, fn: ({speaker, name}) => {
|
||||
transcript = transcript.replaceAll(`[Speaker ${speaker}]`, `[${name}]`);
|
||||
}}
|
||||
]
|
||||
});
|
||||
return transcript;
|
||||
})
|
||||
}
|
||||
|
||||
return Object.assign(p, { abort });
|
||||
private runAsr(file: string, opts: {model?: string, diarization?: boolean} = {}): AbortablePromise<any> {
|
||||
let proc: any;
|
||||
const p = new Promise<any>((resolve, reject) => {
|
||||
this.downloadAsrModel(opts.model).then(m => {
|
||||
if(opts.diarization) {
|
||||
let output = join(Path.dirname(file), 'transcript');
|
||||
proc = spawn(<string>this.ai.options.whisper,
|
||||
['-m', m, '-f', file, '-np', '-ml', '1', '-oj', '-of', output],
|
||||
{stdio: ['ignore', 'ignore', 'pipe']}
|
||||
);
|
||||
proc.on('error', (err: Error) => reject(err));
|
||||
proc.on('close', async (code: number) => {
|
||||
if(code === 0) {
|
||||
output = await fs.readFile(output + '.json', 'utf-8');
|
||||
fs.rm(output + '.json').catch(() => { });
|
||||
try { resolve(JSON.parse(output)); }
|
||||
catch(e) { reject(new Error('Failed to parse whisper JSON')); }
|
||||
} else {
|
||||
reject(new Error(`Exit code ${code}`));
|
||||
}
|
||||
});
|
||||
} else {
|
||||
let output = '';
|
||||
proc = spawn(<string>this.ai.options.whisper, ['-m', m, '-f', file, '-np', '-nt']);
|
||||
proc.on('error', (err: Error) => reject(err));
|
||||
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
||||
proc.on('close', async (code: number) => {
|
||||
if(code === 0) {
|
||||
resolve(output.trim() || null);
|
||||
} else {
|
||||
reject(new Error(`Exit code ${code}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return <any>Object.assign(p, {abort: () => proc?.kill('SIGTERM')});
|
||||
}
|
||||
|
||||
canDiarization = canDiarization;
|
||||
private runDiarization(file: string): AbortablePromise<any> {
|
||||
let aborted = false, abort = () => { aborted = true; };
|
||||
const checkPython = (cmd: string) => {
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const proc = spawn(cmd, ['-W', 'ignore', '-c', 'import pyannote.audio']);
|
||||
proc.on('close', (code: number) => resolve(code === 0));
|
||||
proc.on('error', () => resolve(false));
|
||||
});
|
||||
};
|
||||
const p = Promise.all<any>([
|
||||
checkPython('python'),
|
||||
checkPython('python3'),
|
||||
]).then(<any>(async ([p, p3]: [boolean, boolean]) => {
|
||||
if(aborted) return;
|
||||
if(!p && !p3) throw new Error('Pyannote is not installed: pip install pyannote.audio');
|
||||
const binary = p3 ? 'python3' : 'python';
|
||||
return new Promise((resolve, reject) => {
|
||||
if(aborted) return;
|
||||
let output = '';
|
||||
const proc = spawn(binary, ['-W', 'ignore', '-c', this.pyannote, file]);
|
||||
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
||||
proc.stderr.on('data', (data: Buffer) => console.error(data.toString()));
|
||||
proc.on('close', (code: number) => {
|
||||
if(code === 0) {
|
||||
try { resolve(JSON.parse(output)); }
|
||||
catch (err) { reject(new Error('Failed to parse diarization output')); }
|
||||
} else {
|
||||
reject(new Error(`Python process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
proc.on('error', reject);
|
||||
abort = () => proc.kill('SIGTERM');
|
||||
});
|
||||
}));
|
||||
return <any>Object.assign(p, {abort});
|
||||
}
|
||||
|
||||
asr(path: string, options: { model?: string; diarization?: boolean | 'llm' } = {}): AbortablePromise<string | null> {
|
||||
if(!this.ai.options.whisper) throw new Error('Whisper not configured');
|
||||
|
||||
const tmp = join(mkdtempSync(join(tmpdir(), 'audio-')), 'converted.wav');
|
||||
execSync(`ffmpeg -i "${path}" -ar 16000 -ac 1 -f wav "${tmp}"`, { stdio: 'ignore' });
|
||||
const clean = () => fs.rm(Path.dirname(tmp), {recursive: true, force: true}).catch(() => {});
|
||||
|
||||
if(!options.diarization) return this.runAsr(tmp, {model: options.model});
|
||||
const timestamps = this.runAsr(tmp, {model: options.model, diarization: true});
|
||||
const diarization = this.runDiarization(tmp);
|
||||
let aborted = false, abort = () => {
|
||||
aborted = true;
|
||||
timestamps.abort();
|
||||
diarization.abort();
|
||||
clean();
|
||||
};
|
||||
|
||||
const response = Promise.allSettled([timestamps, diarization]).then(async ([ts, d]) => {
|
||||
if(ts.status == 'rejected') throw new Error('Whisper.cpp timestamps:\n' + ts.reason);
|
||||
if(d.status == 'rejected') throw new Error('Pyannote:\n' + d.reason);
|
||||
if(aborted || !options.diarization) return ts.value;
|
||||
return this.diarizeTranscript(ts.value, d.value, options.diarization == 'llm');
|
||||
}).finally(() => clean());
|
||||
return <any>Object.assign(response, {abort});
|
||||
}
|
||||
|
||||
async downloadAsrModel(model: string = this.whisperModel): Promise<string> {
|
||||
if(!this.ai.options.whisper) throw new Error('Whisper not configured');
|
||||
if(!model.endsWith('.bin')) model += '.bin';
|
||||
const p = Path.join(<string>this.ai.options.path, model);
|
||||
if(await fs.stat(p).then(() => true).catch(() => false)) return p;
|
||||
if(!!this.downloads[model]) return this.downloads[model];
|
||||
this.downloads[model] = fetch(`https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${model}`)
|
||||
.then(resp => resp.arrayBuffer())
|
||||
.then(arr => Buffer.from(arr)).then(async buffer => {
|
||||
await fs.writeFile(p, buffer);
|
||||
delete this.downloads[model];
|
||||
return p;
|
||||
});
|
||||
return this.downloads[model];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { pipeline } from '@xenova/transformers';
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { pipeline } from '@huggingface/transformers';
|
||||
|
||||
let embedder: any;
|
||||
const [modelDir, model] = process.argv.slice(2);
|
||||
|
||||
parentPort?.on('message', async ({text, model, modelDir }) => {
|
||||
if(!embedder) embedder = await pipeline('feature-extraction', 'Xenova/' + model, {quantized: true, cache_dir: modelDir});
|
||||
let text = '';
|
||||
process.stdin.on('data', chunk => text += chunk);
|
||||
process.stdin.on('end', async () => {
|
||||
const embedder = await pipeline('feature-extraction', 'Xenova/' + model, {cache_dir: modelDir});
|
||||
const output = await embedder(text, { pooling: 'mean', normalize: true });
|
||||
const embedding = Array.from(output.data);
|
||||
parentPort?.postMessage({embedding});
|
||||
process.stdout.write(JSON.stringify({embedding}));
|
||||
process.exit();
|
||||
});
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
export * from './ai';
|
||||
export * from './antrhopic';
|
||||
export * from './asr';
|
||||
export * from './audio';
|
||||
export * from './embedder'
|
||||
export * from './llm';
|
||||
export * from './memory';
|
||||
export * from './open-ai';
|
||||
export * from './provider';
|
||||
export * from './tools';
|
||||
|
||||
334
src/kd-tree.ts
Normal file
334
src/kd-tree.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
export type DistanceMetric = "euclidean" | "cosine";
|
||||
|
||||
export interface KDPoint<T = unknown> {
|
||||
vector: number[];
|
||||
payload: T;
|
||||
}
|
||||
|
||||
export interface KNNResult<T = unknown> {
|
||||
point: KDPoint<T>;
|
||||
distance: number;
|
||||
}
|
||||
|
||||
interface KDNode<T> {
|
||||
point: KDPoint<T>;
|
||||
axis: number;
|
||||
left: KDNode<T> | null;
|
||||
right: KDNode<T> | null;
|
||||
}
|
||||
|
||||
// ─── Distance helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
function euclidean(a: number[], b: number[]): number {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const d = a[i] - b[i];
|
||||
sum += d * d;
|
||||
}
|
||||
return Math.sqrt(sum);
|
||||
}
|
||||
|
||||
function cosine(a: number[], b: number[]): number {
|
||||
let dot = 0, normA = 0, normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom === 0 ? 1 : 1 - dot / denom; // distance = 1 - similarity
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps the k closest candidates in memory, evicts the furthest when full
|
||||
*/
|
||||
class BoundedMaxHeap<T> {
|
||||
private heap: KNNResult<T>[] = [];
|
||||
|
||||
constructor(private readonly k: number) {}
|
||||
|
||||
get size(): number { return this.heap.length; }
|
||||
|
||||
get worstDistance(): number {
|
||||
return this.heap.length < this.k ? Infinity : this.heap[0].distance;
|
||||
}
|
||||
|
||||
push(item: KNNResult<T>): void {
|
||||
if (this.heap.length < this.k) {
|
||||
this.heap.push(item);
|
||||
this.bubbleUp(this.heap.length - 1);
|
||||
} else if (item.distance < this.heap[0].distance) {
|
||||
this.heap[0] = item;
|
||||
this.sinkDown(0);
|
||||
}
|
||||
}
|
||||
|
||||
toSortedArray(): KNNResult<T>[] {
|
||||
return [...this.heap].sort((a, b) => a.distance - b.distance);
|
||||
}
|
||||
|
||||
private bubbleUp(i: number): void {
|
||||
while (i > 0) {
|
||||
const parent = (i - 1) >> 1;
|
||||
if (this.heap[parent].distance >= this.heap[i].distance) break;
|
||||
[this.heap[parent], this.heap[i]] = [this.heap[i], this.heap[parent]];
|
||||
i = parent;
|
||||
}
|
||||
}
|
||||
|
||||
private sinkDown(i: number): void {
|
||||
const n = this.heap.length;
|
||||
while (true) {
|
||||
let largest = i;
|
||||
const l = 2 * i + 1, r = 2 * i + 2;
|
||||
if (l < n && this.heap[l].distance > this.heap[largest].distance) largest = l;
|
||||
if (r < n && this.heap[r].distance > this.heap[largest].distance) largest = r;
|
||||
if (largest === i) break;
|
||||
[this.heap[largest], this.heap[i]] = [this.heap[i], this.heap[largest]];
|
||||
i = largest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* K-D Tree for efficient nearest-neighbor search over high-dimensional vectors / embeddings.
|
||||
*
|
||||
* Supports:
|
||||
* - Insertion of labeled points
|
||||
* - k-nearest-neighbor (KNN) search
|
||||
* - Radius search (all points within a given distance)
|
||||
* - Euclidean and cosine distance metrics
|
||||
* - Bulk construction (balanced tree) for best query performance
|
||||
*/
|
||||
export class KDTree<T = unknown> {
|
||||
private root: KDNode<T> | null = null;
|
||||
private _size = 0;
|
||||
private readonly dims: number;
|
||||
private readonly distanceFn: (a: number[], b: number[]) => number;
|
||||
|
||||
/**
|
||||
* @param dims Dimensionality of all vectors (must be consistent).
|
||||
* @param metric Distance metric to use. Default: "euclidean".
|
||||
* @param points Optional initial set of points. Builds a balanced tree
|
||||
* in O(n log² n) — prefer this over inserting one-by-one
|
||||
* when you have a large corpus.
|
||||
*/
|
||||
constructor(
|
||||
dims: number,
|
||||
metric: DistanceMetric = "euclidean",
|
||||
points?: KDPoint<T>[]
|
||||
) {
|
||||
this.dims = dims;
|
||||
this.distanceFn = metric === "cosine" ? cosine : euclidean;
|
||||
|
||||
if (points && points.length > 0) {
|
||||
this.validateAll(points);
|
||||
this.root = this.buildBalanced([...points], 0);
|
||||
this._size = points.length;
|
||||
}
|
||||
}
|
||||
|
||||
/** Total number of points stored in the tree. */
|
||||
get size(): number { return this._size; }
|
||||
|
||||
// ── Insertion ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Insert a single point. O(log n) average, O(n) worst case on skewed data.
|
||||
* For bulk loading prefer passing points to the constructor.
|
||||
*/
|
||||
insert(point: KDPoint<T>): void {
|
||||
this.validate(point);
|
||||
this.root = this.insertNode(this.root, point, 0);
|
||||
this._size++;
|
||||
}
|
||||
|
||||
// ── KNN search ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Find the k nearest neighbors to `query`.
|
||||
* Returns results sorted by distance ascending.
|
||||
*/
|
||||
knn(query: number[], k: number): KNNResult<T>[] {
|
||||
if (k <= 0) throw new RangeError("k must be a positive integer");
|
||||
this.validateVector(query);
|
||||
|
||||
const heap = new BoundedMaxHeap<T>(k);
|
||||
this.searchKNN(this.root, query, k, heap, 0);
|
||||
return heap.toSortedArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Nearest single neighbor. Convenience wrapper around knn(query, 1).
|
||||
* Returns null if the tree is empty.
|
||||
*/
|
||||
nearest(query: number[]): KNNResult<T> | null {
|
||||
const results = this.knn(query, 1);
|
||||
return results[0] ?? null;
|
||||
}
|
||||
|
||||
// ── Radius search ──────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Return all points whose distance to `query` is ≤ `radius`,
|
||||
* sorted by distance ascending.
|
||||
*/
|
||||
radiusSearch(query: number[], radius: number): KNNResult<T>[] {
|
||||
if (radius < 0) throw new RangeError("radius must be non-negative");
|
||||
this.validateVector(query);
|
||||
|
||||
const results: KNNResult<T>[] = [];
|
||||
this.searchRadius(this.root, query, radius, results, 0);
|
||||
results.sort((a, b) => a.distance - b.distance);
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Conversion ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Collect all points in the tree (order not guaranteed). */
|
||||
toArray(): KDPoint<T>[] {
|
||||
const out: KDPoint<T>[] = [];
|
||||
this.collect(this.root, out);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the tree from its current points as a balanced tree.
|
||||
* Useful after many individual insertions to restore O(log n) query time.
|
||||
*/
|
||||
rebalance(): void {
|
||||
const points = this.toArray();
|
||||
this.root = points.length ? this.buildBalanced(points, 0) : null;
|
||||
}
|
||||
|
||||
// ── Private: build ─────────────────────────────────────────────────────────
|
||||
|
||||
private buildBalanced(points: KDPoint<T>[], depth: number): KDNode<T> {
|
||||
const axis = depth % this.dims;
|
||||
points.sort((a, b) => a.vector[axis] - b.vector[axis]);
|
||||
|
||||
const mid = Math.floor(points.length / 2);
|
||||
return {
|
||||
point: points[mid],
|
||||
axis,
|
||||
left: points.slice(0, mid).length
|
||||
? this.buildBalanced(points.slice(0, mid), depth + 1)
|
||||
: null,
|
||||
right: points.slice(mid + 1).length
|
||||
? this.buildBalanced(points.slice(mid + 1), depth + 1)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Private: insert ────────────────────────────────────────────────────────
|
||||
|
||||
private insertNode(
|
||||
node: KDNode<T> | null,
|
||||
point: KDPoint<T>,
|
||||
depth: number
|
||||
): KDNode<T> {
|
||||
if (node === null) {
|
||||
return { point, axis: depth % this.dims, left: null, right: null };
|
||||
}
|
||||
const axis = depth % this.dims;
|
||||
if (point.vector[axis] < node.point.vector[axis]) {
|
||||
node.left = this.insertNode(node.left, point, depth + 1);
|
||||
} else {
|
||||
node.right = this.insertNode(node.right, point, depth + 1);
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
// ── Private: KNN traversal ─────────────────────────────────────────────────
|
||||
|
||||
private searchKNN(
|
||||
node: KDNode<T> | null,
|
||||
query: number[],
|
||||
k: number,
|
||||
heap: BoundedMaxHeap<T>,
|
||||
depth: number
|
||||
): void {
|
||||
if (node === null) return;
|
||||
|
||||
const dist = this.distanceFn(query, node.point.vector);
|
||||
heap.push({ point: node.point, distance: dist });
|
||||
|
||||
const axis = node.axis;
|
||||
const diff = query[axis] - node.point.vector[axis];
|
||||
const [near, far] = diff <= 0
|
||||
? [node.left, node.right]
|
||||
: [node.right, node.left];
|
||||
|
||||
this.searchKNN(near, query, k, heap, depth + 1);
|
||||
|
||||
// Only explore the far side if it could contain a closer point.
|
||||
// For cosine distance we can't prune by axis gap alone, so always explore.
|
||||
const shouldExplore =
|
||||
this.distanceFn === cosine
|
||||
? true
|
||||
: Math.abs(diff) < heap.worstDistance;
|
||||
|
||||
if (shouldExplore) {
|
||||
this.searchKNN(far, query, k, heap, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private: radius traversal ──────────────────────────────────────────────
|
||||
|
||||
private searchRadius(
|
||||
node: KDNode<T> | null,
|
||||
query: number[],
|
||||
radius: number,
|
||||
results: KNNResult<T>[],
|
||||
depth: number
|
||||
): void {
|
||||
if (node === null) return;
|
||||
|
||||
const dist = this.distanceFn(query, node.point.vector);
|
||||
if (dist <= radius) {
|
||||
results.push({ point: node.point, distance: dist });
|
||||
}
|
||||
|
||||
const axis = node.axis;
|
||||
const diff = query[axis] - node.point.vector[axis];
|
||||
const [near, far] = diff <= 0
|
||||
? [node.left, node.right]
|
||||
: [node.right, node.left];
|
||||
|
||||
this.searchRadius(near, query, radius, results, depth + 1);
|
||||
|
||||
const shouldExplore =
|
||||
this.distanceFn === cosine ? true : Math.abs(diff) <= radius;
|
||||
|
||||
if (shouldExplore) {
|
||||
this.searchRadius(far, query, radius, results, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private: collect ───────────────────────────────────────────────────────
|
||||
|
||||
private collect(node: KDNode<T> | null, out: KDPoint<T>[]): void {
|
||||
if (node === null) return;
|
||||
out.push(node.point);
|
||||
this.collect(node.left, out);
|
||||
this.collect(node.right, out);
|
||||
}
|
||||
|
||||
// ── Private: validation ────────────────────────────────────────────────────
|
||||
|
||||
private validateVector(v: number[]): void {
|
||||
if (v.length !== this.dims) {
|
||||
throw new TypeError(
|
||||
`Vector length ${v.length} does not match tree dimensionality ${this.dims}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private validate(point: KDPoint<T>): void {
|
||||
this.validateVector(point.vector);
|
||||
}
|
||||
|
||||
private validateAll(points: KDPoint<T>[]): void {
|
||||
for (const p of points) this.validate(p);
|
||||
}
|
||||
}
|
||||
552
src/llm.ts
552
src/llm.ts
@@ -1,17 +1,30 @@
|
||||
import {JSONAttemptParse} from '@ztimson/utils';
|
||||
import {snakeCase} from '@ztimson/utils';
|
||||
import {AbortablePromise, Ai} from './ai.ts';
|
||||
import {Anthropic} from './antrhopic.ts';
|
||||
import {OpenAi} from './open-ai.ts';
|
||||
import {LLMProvider} from './provider.ts';
|
||||
import {AiTool} from './tools.ts';
|
||||
import {Worker} from 'worker_threads';
|
||||
import {AiTool, AiToolArg} from './tools.ts';
|
||||
import {fileURLToPath} from 'url';
|
||||
import {dirname, join} from 'path';
|
||||
import {spawn} from 'node:child_process';
|
||||
import {Memory, MemoryCache, MemoryManager, MemoryOptions} from './memory.ts';
|
||||
|
||||
export type AnthropicConfig = {proto: 'anthropic', token: string};
|
||||
export type OllamaConfig = {proto: 'ollama', host: string};
|
||||
export type OpenAiConfig = {proto: 'openai', host?: string, token: string};
|
||||
|
||||
export type Agent = {
|
||||
name: string;
|
||||
description?: string;
|
||||
model?: string | null;
|
||||
temperature?: number;
|
||||
system: string;
|
||||
delegate?: boolean;
|
||||
skills?: Skill[] | null;
|
||||
tools?: AiTool[] | null;
|
||||
mcp?: McpServer[] | null;
|
||||
agents?: string[] | null;
|
||||
}
|
||||
|
||||
export type LLMMessage = {
|
||||
/** Message originator */
|
||||
role: 'assistant' | 'system' | 'user';
|
||||
@@ -36,19 +49,9 @@ export type LLMMessage = {
|
||||
timestamp?: number;
|
||||
}
|
||||
|
||||
/** Background information the AI will be fed */
|
||||
export type LLMMemory = {
|
||||
/** What entity is this fact about */
|
||||
owner: string;
|
||||
/** The information that will be remembered */
|
||||
fact: string;
|
||||
/** Owner and fact embedding vector */
|
||||
embeddings: [number[], number[]];
|
||||
/** Creation time */
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
export type LLMRequest = {
|
||||
/** Return a parsed JSON object that matches the schema */
|
||||
schema?: AiToolArg;
|
||||
/** System prompt */
|
||||
system?: string;
|
||||
/** Message history */
|
||||
@@ -64,17 +67,44 @@ export type LLMRequest = {
|
||||
/** Stream response */
|
||||
stream?: (chunk: {text?: string, tool?: string, done?: true}) => any;
|
||||
/** Compress old messages in the chat to free up context */
|
||||
compress?: {
|
||||
/** Trigger chat compression once context exceeds the token count */
|
||||
max: number;
|
||||
/** Compress chat until context size smaller than */
|
||||
min: number
|
||||
},
|
||||
/** Background information the AI will be fed */
|
||||
memory?: LLMMemory[],
|
||||
compress?: {max: number; min: number};
|
||||
/** User's memory documents - RAG injected automatically each turn */
|
||||
memory?: Memory[] | MemoryCache | MemoryOptions;
|
||||
/** Model to use for memory operations */
|
||||
memoryModel?: string;
|
||||
/** Skill documents the AI can browse and read on demand */
|
||||
skills?: Skill[];
|
||||
/** MCP servers to connect and expose as tools */
|
||||
mcp?: McpServer[];
|
||||
/** Subagents exposed as delegatable/wrapped tools */
|
||||
agents?: Agent[];
|
||||
/** @internal recursion guard for nested agent delegation */
|
||||
_agentDepth?: number;
|
||||
}
|
||||
|
||||
export type McpServer = {
|
||||
/** MCP server name for humans */
|
||||
name: string;
|
||||
/** Host URL */
|
||||
host: string;
|
||||
/** Server access token */
|
||||
token?: string;
|
||||
}
|
||||
|
||||
export type Skill = {
|
||||
/** Name of skill for humans */
|
||||
name: string;
|
||||
/** Description LLM will use to decide to learn a skill */
|
||||
description: string;
|
||||
/** Skill instructions */
|
||||
content: string;
|
||||
}
|
||||
|
||||
const MAX_AGENT_DEPTH = 5;
|
||||
|
||||
class LLM {
|
||||
private memoryManager!: MemoryManager;
|
||||
|
||||
defaultModel!: string;
|
||||
models: {[model: string]: LLMProvider} = {};
|
||||
|
||||
@@ -83,85 +113,251 @@ class LLM {
|
||||
Object.entries(ai.options.llm.models).forEach(([model, config]) => {
|
||||
if(!this.defaultModel) this.defaultModel = model;
|
||||
if(config.proto == 'anthropic') this.models[model] = new Anthropic(this.ai, config.token, model);
|
||||
else if(config.proto == 'ollama') this.models[model] = new OpenAi(this.ai, config.host, 'not-needed', model);
|
||||
else if(config.proto == 'openai') this.models[model] = new OpenAi(this.ai, config.host || null, config.token, model);
|
||||
});
|
||||
this.memoryManager = new MemoryManager(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chat with LLM
|
||||
* @param {string} message Question
|
||||
* @param {LLMRequest} options Configuration options and chat history
|
||||
* @returns {{abort: () => void, response: Promise<string>}} Function to abort response and chat history
|
||||
* Wrap agents as tools. Nested delegation is opt-in only (empty by default, like
|
||||
* tools/skills/mcp) and an agent can never call itself even if explicitly whitelisted.
|
||||
* Delegate results are queued in `pending` and spliced into history by `ask()` after
|
||||
* the provider's own end-of-turn history sync has already run.
|
||||
*/
|
||||
private setupAgent(agents: Agent[] = [], allAgents: Agent[], pending: Map<string, any>, aborts: (() => void)[], depth = 0): AiTool[] {
|
||||
return agents.map(a => {
|
||||
const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`;
|
||||
return {
|
||||
name: toolName,
|
||||
description: `${a.delegate ? 'Delegate to ' : ''}Subagent: ${a.description || a.name}`,
|
||||
args: {
|
||||
context: {type: 'string', description: 'Summary of related messages, samples, files, etc...', required: true},
|
||||
instructions: {type: 'string', description: 'Detailed instructions for subagent to complete', required: true},
|
||||
},
|
||||
fn: async (args: any, stream: any, ai: any, id?: string) => {
|
||||
if(depth >= MAX_AGENT_DEPTH) return 'Max agent delegation depth exceeded';
|
||||
const subHistory: LLMMessage[] = [];
|
||||
|
||||
// Opt-in only, self always excluded regardless of whitelist
|
||||
const nested = (a.agents || [])
|
||||
.map(name => allAgents.find(x => x.name === name))
|
||||
.filter((x): x is Agent => !!x && x.name !== a.name);
|
||||
|
||||
const request = this.ask(`${args.instructions}${args.context ? `\n\n<context>${args.context}</context>` : ''}`, {
|
||||
system: `You are a specialized subagent. ${a.delegate ? 'Your output streams directly to the user for the remainder of this turn.' : 'You are wrapped in a tool call that will be analysis by an LLM'}
|
||||
As a subagent, focus on executing your task completely using available tools and returning only the final result - no commentary, questions, or dialogue.
|
||||
|
||||
${a.system}`,
|
||||
model: a.model || undefined,
|
||||
temperature: a.temperature,
|
||||
stream: a.delegate ? stream : undefined,
|
||||
history: subHistory,
|
||||
mcp: a.mcp || undefined,
|
||||
skills: a.skills || undefined,
|
||||
tools: a.tools || undefined,
|
||||
agents: nested,
|
||||
_agentDepth: depth + 1,
|
||||
} as any);
|
||||
aborts.push(request.abort);
|
||||
const resp = await request;
|
||||
|
||||
if(a.delegate) {
|
||||
pending.set(<string>id, {resp, subHistory});
|
||||
return '';
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private async setupMcp(servers: McpServer[] = []): Promise<{prompt: string, tools: AiTool[]}> {
|
||||
if(!servers?.length) return {prompt: '', tools: []};
|
||||
const allTools: AiTool[] = [];
|
||||
await Promise.all(servers.map(async server => {
|
||||
const res = await fetch(`${server.host}/tools`, {headers: server.token ? {Authorization: `Bearer ${server.token}`} : {}});
|
||||
const mcp: any = await res.json();
|
||||
if(!mcp?.tools) return;
|
||||
for(const t of mcp.tools) {
|
||||
const args: Record<string, any> = {};
|
||||
if(t.inputSchema?.properties) {
|
||||
for(const [key, val] of Object.entries<any>(t.inputSchema.properties)) {
|
||||
args[key] = {type: val.type || 'string', description: val.description || '', required: t.inputSchema.required?.includes(key)};
|
||||
}
|
||||
}
|
||||
allTools.push({
|
||||
name: `${server.name}_${t.name}`,
|
||||
description: t.description || '',
|
||||
args,
|
||||
fn: async (a: any) => {
|
||||
const r = await fetch(`${server.host}/tools/call`, {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json', ...(server.token ? {Authorization: `Bearer ${server.token}`} : {})},
|
||||
body: JSON.stringify({name: t.name, arguments: a})
|
||||
});
|
||||
const data: any = await r.json();
|
||||
return data?.content?.[0]?.text ?? JSON.stringify(data);
|
||||
}
|
||||
});
|
||||
}
|
||||
}));
|
||||
|
||||
const list = allTools.map(t => `- ${t.name}: ${t.description}`).join('\n');
|
||||
return {
|
||||
prompt: `You have access to the following MCP tools:\n${list}`,
|
||||
tools: allTools
|
||||
};
|
||||
}
|
||||
|
||||
private setupSkills(skills: Skill[] = []): {prompt: string, tools: AiTool[]} {
|
||||
if(!skills?.length) return {prompt: '', tools: []};
|
||||
const list = skills.map(s => `- ${s.name}: ${s.description}`).join('\n');
|
||||
return {
|
||||
prompt: `You have access to the following skill documents, use \`read_skill\` to access them:\n${list}`,
|
||||
tools: [{
|
||||
name: 'skill_read',
|
||||
description: 'Read the full content of a skill/knowledge document',
|
||||
args: {
|
||||
name: {type: 'string', description: 'Exact skill name', required: true}
|
||||
},
|
||||
fn: (args: any) => {
|
||||
const skill = skills.find(s => s.name === args.name);
|
||||
if(!skill) return `Skill not found. Available:\n${list}`;
|
||||
return `# ${skill.name}\n${skill.content}`;
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
ask(message: string, options: LLMRequest = {}): AbortablePromise<string> {
|
||||
options = <any>{
|
||||
system: '',
|
||||
...this.ai.options.llm,
|
||||
models: undefined,
|
||||
history: [],
|
||||
...options,
|
||||
}
|
||||
const m = options.model || this.defaultModel;
|
||||
if(!this.models[m]) throw new Error(`Model does not exist: ${m}`);
|
||||
let abort = () => {};
|
||||
return Object.assign(new Promise<string>(async res => {
|
||||
if(!options.history) options.history = [];
|
||||
// If memories were passed, find any relivant ones and add a tool for ADHOC lookups
|
||||
if(options.memory) {
|
||||
options.system = (options.system || '') + '\nYou have passive persistent memory never make any mention of your memory capabilities and what you can/cannot remember\n';
|
||||
const search = async (query?: string | null, subject?: string | null, limit = 50) => {
|
||||
const [o, q] = await Promise.all([
|
||||
subject ? this.embedding(subject) : Promise.resolve(null),
|
||||
query ? this.embedding(query) : Promise.resolve(null),
|
||||
]);
|
||||
return (options.memory || [])
|
||||
.map(m => ({...m, score: o ? this.cosineSimilarity(m.embeddings[0], o[0].embedding) : 1}))
|
||||
.filter((m: any) => m.score >= 0.8)
|
||||
.map((m: any) => ({...m, score: q ? this.cosineSimilarity(m.embeddings[1], q[0].embedding) : m.score}))
|
||||
.filter((m: any) => m.score >= 0.2)
|
||||
.toSorted((a: any, b: any) => a.score - b.score)
|
||||
.slice(0, limit);
|
||||
let request: AbortablePromise<string> | null = null;
|
||||
let aborted = false;
|
||||
const nestedAborts: (() => void)[] = [];
|
||||
const abort = () => {
|
||||
aborted = true;
|
||||
request?.abort?.();
|
||||
nestedAborts.forEach(a => a());
|
||||
};
|
||||
|
||||
const promise = (async () => {
|
||||
let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
|
||||
const prompts: string[] = [];
|
||||
let history = options.history || [];
|
||||
|
||||
// MCP
|
||||
const mcp = options.mcp || this.ai.options?.llm?.mcp;
|
||||
if(mcp?.length) {
|
||||
const m = await this.setupMcp(mcp);
|
||||
prompts.unshift(m.prompt);
|
||||
tools.push(...m.tools);
|
||||
}
|
||||
|
||||
const relevant = await search(message);
|
||||
if(relevant.length) options.history.push({role: 'assistant', content: 'Things I remembered:\n' + relevant.map(m => `${m.owner}: ${m.fact}`).join('\n')});
|
||||
options.tools = [...options.tools || [], {
|
||||
name: 'read_memory',
|
||||
description: 'Check your long-term memory for more information',
|
||||
args: {
|
||||
subject: {type: 'string', description: 'Find information by a subject topic, can be used with or without query argument'},
|
||||
query: {type: 'string', description: 'Search memory based on a query, can be used with or without subject argument'},
|
||||
limit: {type: 'number', description: 'Result limit, default 5'},
|
||||
},
|
||||
fn: (args) => {
|
||||
if(!args.subject && !args.query) throw new Error('Either a subject or query argument is required');
|
||||
return search(args.query, args.subject, args.limit || 5);
|
||||
}
|
||||
}];
|
||||
// Skills
|
||||
const skills = options.skills || this.ai.options?.llm?.skills;
|
||||
if(skills?.length) {
|
||||
const s = this.setupSkills(skills);
|
||||
prompts.unshift(s.prompt);
|
||||
tools.push(...s.tools);
|
||||
}
|
||||
|
||||
// Ask
|
||||
const resp = await this.models[m].ask(message, options);
|
||||
// Agents
|
||||
const agents = options.agents || this.ai.options?.llm?.agents;
|
||||
const pendingDelegates = new Map<string, any>();
|
||||
if(agents?.length) tools.push(...this.setupAgent(agents, agents, pendingDelegates, nestedAborts, options._agentDepth || 0));
|
||||
|
||||
// Remove any memory calls
|
||||
if(options.memory) {
|
||||
const i = options.history?.findIndex((h: any) => h.role == 'assistant' && h.content.startsWith('Things I remembered:'));
|
||||
if(i != null && i >= 0) options.history?.splice(i, 1);
|
||||
// Memory
|
||||
const mem = MemoryManager.normalize(options.memory);
|
||||
if(mem) {
|
||||
const mems = mem.memory instanceof MemoryCache ? mem.memory.memories : mem.memory;
|
||||
if(mems.length) {
|
||||
if(mem.inject) {
|
||||
const pool = 15; // candidates considered, cheap since only refs are listed
|
||||
const budget = mem.maxTokens ?? 2000; // actual content injected
|
||||
const relevant = await this.memoryManager.recollect(message, mem.memory, pool);
|
||||
|
||||
let used = 0;
|
||||
const preloaded: typeof relevant = [];
|
||||
const listed: typeof relevant = [];
|
||||
for(const r of relevant) {
|
||||
const t = this.estimateTokens(r.content);
|
||||
if(used + t <= budget || preloaded.length === 0) {
|
||||
preloaded.push(r);
|
||||
used += t;
|
||||
} else listed.push(r);
|
||||
}
|
||||
|
||||
// Handle compression and memory extraction
|
||||
if(options.compress || options.memory) {
|
||||
let compressed = null;
|
||||
if(options.compress) {
|
||||
compressed = await this.ai.language.compressHistory(options.history, options.compress.max, options.compress.min, options);
|
||||
options.history.splice(0, options.history.length, ...compressed.history);
|
||||
} else {
|
||||
const i = options.history?.findLastIndex(m => m.role == 'user') ?? -1;
|
||||
compressed = await this.ai.language.compressHistory(i != -1 ? options.history.slice(i) : options.history, 0, 0, options);
|
||||
prompts.unshift(`You have access to the following memory files:
|
||||
${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')}
|
||||
${preloaded.length ? `
|
||||
Relevant memories have been preloaded:
|
||||
${preloaded.map(r => `
|
||||
**${r.name}**
|
||||
${r.description}
|
||||
${r.content}
|
||||
`).join('\n---\n')}
|
||||
` : ''}${listed.length ? `
|
||||
Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.name).join(', ')}
|
||||
` : ''}`.trim());
|
||||
}
|
||||
if(options.memory) {
|
||||
const updated = options.memory
|
||||
.filter(m => !compressed.memory.some(m2 => this.cosineSimilarity(m.embeddings[1], m2.embeddings[1]) > 0.8))
|
||||
.concat(compressed.memory);
|
||||
options.memory.splice(0, options.memory.length, ...updated);
|
||||
if(mem.tool) tools.push(this.memoryManager.tools.read(mem.memory));
|
||||
}
|
||||
}
|
||||
return res(resp);
|
||||
}), {abort});
|
||||
|
||||
if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
|
||||
|
||||
prompts.unshift(options.system || this.ai.options.llm?.system || '');
|
||||
request = this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
|
||||
let resp = await request;
|
||||
|
||||
// Spice delegated agents response into history
|
||||
let lastDelegateResp: string | null = null;
|
||||
if(pendingDelegates.size) {
|
||||
for(let i = 0; i < history.length; i++) {
|
||||
const h: any = history[i];
|
||||
if(h.role !== 'tool' || !pendingDelegates.has(h.id)) continue;
|
||||
const {resp: delegateResp, subHistory} = pendingDelegates.get(h.id)!;
|
||||
pendingDelegates.delete(h.id);
|
||||
const insert: LLMMessage[] = [...subHistory.filter(sh => sh.role === 'tool'), {role: 'assistant', content: delegateResp, timestamp: Date.now()}];
|
||||
history.splice(i + 1, 0, ...insert);
|
||||
lastDelegateResp = delegateResp;
|
||||
i += insert.length;
|
||||
}
|
||||
}
|
||||
|
||||
// If the orchestrator added no commentary of its own, its answer IS the delegate's answer
|
||||
if(typeof resp === 'string' && !resp.trim() && lastDelegateResp !== null) resp = lastDelegateResp;
|
||||
|
||||
// Trim memory injections from history
|
||||
if(mem?.tool) history.splice(0, history.length, ...history.filter(h => h.role !== 'tool' || h.name !== 'memory_recall'));
|
||||
|
||||
// Auto-memorize before compressing
|
||||
if(options.compress && this.estimateTokens(history) >= options.compress.max) {
|
||||
if(mem?.update) await this.memoryManager.memorize(history, mem.memory, {model: options.memoryModel || this.defaultModel, ...options});
|
||||
const compressed = await this.compressHistory(history, options.compress.max, options.compress.min, options);
|
||||
if(options.history) options.history.splice(0, options.history.length, ...compressed);
|
||||
}
|
||||
|
||||
return resp;
|
||||
})();
|
||||
|
||||
return Object.assign(promise, {abort});
|
||||
}
|
||||
|
||||
/**
|
||||
* Digest full conversation history into memory documents.
|
||||
* Call on session end to persist the conversation.
|
||||
*/
|
||||
async updateMemory(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest = {}): Promise<Memory[]> {
|
||||
return this.memoryManager.memorize(history, memories, {model: this.defaultModel, ...options});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -172,32 +368,24 @@ class LLM {
|
||||
* @param {LLMRequest} options LLM options
|
||||
* @returns {Promise<LLMMessage[]>} New chat history will summary at index 0
|
||||
*/
|
||||
async compressHistory(history: LLMMessage[], max: number, min: number, options?: LLMRequest): Promise<{history: LLMMessage[], memory: LLMMemory[]}> {
|
||||
if(this.estimateTokens(history) < max) return {history, memory: []};
|
||||
async compressHistory(history: LLMMessage[], max: number, min: number, options?: LLMRequest): Promise<LLMMessage[]> {
|
||||
if(this.estimateTokens(history) < max) return history;
|
||||
let keep = 0, tokens = 0;
|
||||
for(let m of history.toReversed()) {
|
||||
tokens += this.estimateTokens(m.content);
|
||||
if(tokens < min) keep++;
|
||||
else break;
|
||||
}
|
||||
if(history.length <= keep) return {history, memory: []};
|
||||
if(history.length <= keep) return history;
|
||||
const system = history[0].role == 'system' ? history[0] : null,
|
||||
recent = keep == 0 ? [] : history.slice(-keep),
|
||||
process = (keep == 0 ? history : history.slice(0, -keep)).filter(h => h.role === 'assistant' || h.role === 'user');
|
||||
|
||||
const summary: any = await this.json(process.map(m => `${m.role}: ${m.content}`).join('\n\n'), '{summary: string, facts: [[subject, fact]]}', {
|
||||
system: 'Create the smallest summary possible, no more than 500 tokens. Create a list of NEW facts (split by subject [pro]noun and fact) about what you learned from this conversation that you didn\'t already know or get from a tool call or system prompt. Focus only on new information about people, topics, or facts. Avoid generating facts about the AI.',
|
||||
model: options?.model,
|
||||
temperature: options?.temperature || 0.3
|
||||
});
|
||||
const timestamp = new Date();
|
||||
const memory = await Promise.all((summary?.facts || [])?.map(async ([owner, fact]: [string, string]) => {
|
||||
const e = await Promise.all([this.embedding(owner), this.embedding(`${owner}: ${fact}`)]);
|
||||
return {owner, fact, embeddings: [e[0][0].embedding, e[1][0].embedding], timestamp};
|
||||
}));
|
||||
const h = [{role: 'assistant', content: `Conversation Summary: ${summary?.summary}`, timestamp: Date.now()}, ...recent];
|
||||
const summary: any = await this.summarize(process.map(m => `[${m.role}]: ${m.content}`).join('\n\n'), 500, options);
|
||||
const d = Date.now();
|
||||
const h = [{role: <any>'tool', name: 'summary', id: `summary_` + d, args: {}, content: `Conversation Summary: ${summary?.summary}`, timestamp: d}, ...recent];
|
||||
if(system) h.splice(0, 0, system);
|
||||
return {history: <any>h, memory};
|
||||
return h;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -234,7 +422,7 @@ class LLM {
|
||||
return `${p}: ${Array.isArray(value) ? value.join(', ') : value}`;
|
||||
});
|
||||
};
|
||||
const lines = typeof target === 'object' ? objString(target) : target.split('\n');
|
||||
const lines = typeof target === 'object' ? objString(target) : target.toString().split('\n');
|
||||
const tokens = lines.flatMap(l => [...l.split(/\s+/).filter(Boolean), '\n']);
|
||||
const chunks: string[] = [];
|
||||
for(let i = 0; i < tokens.length;) {
|
||||
@@ -255,37 +443,55 @@ class LLM {
|
||||
/**
|
||||
* Create a vector representation of a string
|
||||
* @param {object | string} target Item that will be embedded (objects get converted)
|
||||
* @param {number} maxTokens Chunking size. More = better context, less = more specific (Search by paragraphs or lines)
|
||||
* @param {number} overlapTokens Includes previous X tokens to provide continuity to AI (In addition to max tokens)
|
||||
* @param {maxTokens?: number, overlapTokens?: number} opts Options for embedding such as chunk sizes
|
||||
* @returns {Promise<Awaited<{index: number, embedding: number[], text: string, tokens: number}>[]>} Chunked embeddings
|
||||
*/
|
||||
embedding(target: object | string, maxTokens = 500, overlapTokens = 50) {
|
||||
embedding(target: object | string, opts: {maxTokens?: number, overlapTokens?: number} = {}): AbortablePromise<{index: number, embedding: number[], text: string, tokens: number}[]> {
|
||||
let {maxTokens = 500, overlapTokens = 50} = opts;
|
||||
let aborted = false;
|
||||
const abort = () => { aborted = true; };
|
||||
|
||||
const embed = (text: string): Promise<number[]> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'embedder.js'));
|
||||
const handleMessage = ({ embedding }: any) => {
|
||||
worker.terminate();
|
||||
resolve(embedding);
|
||||
};
|
||||
const handleError = (err: Error) => {
|
||||
worker.terminate();
|
||||
if(aborted) return reject(new Error('Aborted'));
|
||||
const args: string[] = [
|
||||
join(dirname(fileURLToPath(import.meta.url)), 'embedder.js'),
|
||||
<string>this.ai.options.path,
|
||||
this.ai.options?.embedder || 'bge-small-en-v1.5'
|
||||
];
|
||||
const proc = spawn('node', args, {stdio: ['pipe', 'pipe', 'ignore']});
|
||||
proc.stdin.write(text);
|
||||
proc.stdin.end();
|
||||
let output = '';
|
||||
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
||||
proc.on('close', (code: number) => {
|
||||
if(aborted) return reject(new Error('Aborted'));
|
||||
if(code === 0) {
|
||||
try {
|
||||
const result = JSON.parse(output);
|
||||
resolve(result.embedding);
|
||||
} catch(err) {
|
||||
reject(err);
|
||||
};
|
||||
worker.on('message', handleMessage);
|
||||
worker.on('error', handleError);
|
||||
worker.on('exit', (code) => {
|
||||
if(code !== 0) reject(new Error(`Worker exited with code ${code}`));
|
||||
}
|
||||
} else {
|
||||
reject(new Error(`Embedder process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
worker.postMessage({text, model: this.ai.options?.embedder || 'bge-small-en-v1.5', modelDir: this.ai.options.path});
|
||||
proc.on('error', reject);
|
||||
});
|
||||
};
|
||||
const chunks = this.chunk(target, maxTokens, overlapTokens);
|
||||
return Promise.all(chunks.map(async (text, index) => ({
|
||||
index,
|
||||
embedding: await embed(text),
|
||||
text,
|
||||
tokens: this.estimateTokens(text),
|
||||
})));
|
||||
|
||||
const p = (async () => {
|
||||
const chunks = this.chunk(target, maxTokens, overlapTokens), results: any[] = [];
|
||||
for(let i = 0; i < chunks.length; i++) {
|
||||
if(aborted) break;
|
||||
const text = chunks[i];
|
||||
const embedding = await embed(text);
|
||||
results.push({index: i, embedding, text, tokens: this.estimateTokens(text)});
|
||||
}
|
||||
return results;
|
||||
})();
|
||||
return <any>Object.assign(p, {abort});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,44 +510,90 @@ class LLM {
|
||||
* @param {string} searchTerms Multiple search terms to check against target
|
||||
* @returns {{avg: number, max: number, similarities: number[]}} Similarity values 0-1: 0 = unique, 1 = identical
|
||||
*/
|
||||
fuzzyMatch(target: string, ...searchTerms: string[]) {
|
||||
fuzzyMatch(target, ...searchTerms) {
|
||||
if (searchTerms.length < 2) throw new Error('Requires at least 2 strings to compare');
|
||||
const vector = (text: string, dimensions: number = 10): number[] => {
|
||||
return text.toLowerCase().split('').map((char, index) =>
|
||||
(char.charCodeAt(0) * (index + 1)) % dimensions / dimensions).slice(0, dimensions);
|
||||
const levenshtein = (a, b) => {
|
||||
const m = a.length, n = b.length;
|
||||
if (!m) return n;
|
||||
if (!n) return m;
|
||||
const dp = Array.from({length: m + 1}, (_, i) => [i, ...Array(n).fill(0)]);
|
||||
for (let j = 0; j <= n; j++) dp[0][j] = j;
|
||||
for (let i = 1; i <= m; i++) {
|
||||
for (let j = 1; j <= n; j++) {
|
||||
dp[i][j] = a[i - 1] === b[j - 1]
|
||||
? dp[i - 1][j - 1]
|
||||
: 1 + Math.min(dp[i - 1][j - 1], dp[i - 1][j], dp[i][j - 1]);
|
||||
}
|
||||
const v = vector(target);
|
||||
const similarities = searchTerms.map(t => vector(t)).map(refVector => this.cosineSimilarity(v, refVector))
|
||||
return {avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length, max: Math.max(...similarities), similarities}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask a question with JSON response
|
||||
* @param {string} text Text to process
|
||||
* @param {string} schema JSON schema the AI should match
|
||||
* @param {LLMRequest} options Configuration options and chat history
|
||||
* @returns {Promise<{} | {} | RegExpExecArray | null>}
|
||||
*/
|
||||
async json(text: string, schema: string, options?: LLMRequest): Promise<any> {
|
||||
let resp = await this.ask(text, {...options, system: (options?.system ? `${options.system}\n` : '') + `Only respond using a JSON code block matching this schema:
|
||||
\`\`\`json
|
||||
${schema}
|
||||
\`\`\``});
|
||||
if(!resp) return {};
|
||||
const codeBlock = /```(?:.+)?\s*([\s\S]*?)```/.exec(resp);
|
||||
const jsonStr = codeBlock ? codeBlock[1].trim() : resp;
|
||||
return JSONAttemptParse(jsonStr, {});
|
||||
return dp[m][n];
|
||||
};
|
||||
const similarity = (a, b) => {
|
||||
a = a.toLowerCase(); b = b.toLowerCase();
|
||||
return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1);
|
||||
};
|
||||
const similarities = searchTerms.map(t => similarity(target, t));
|
||||
return {
|
||||
avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length,
|
||||
max: Math.max(...similarities),
|
||||
similarities
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a summary of some text
|
||||
* @param {string} text Text to summarize
|
||||
* @param {number} tokens Max number of tokens
|
||||
* @param {number} length Max number of words
|
||||
* @param options LLM request options
|
||||
* @returns {Promise<string>} Summary
|
||||
*/
|
||||
summarize(text: string, tokens: number, options?: LLMRequest): Promise<string | null> {
|
||||
return this.ask(text, {system: `Generate a brief summary <= ${tokens} tokens. Output nothing else`, temperature: 0.3, ...options});
|
||||
async summarize(text: string, length: number = 500, options?: LLMRequest): Promise<string | null> {
|
||||
let system = `Your job is to summarize the users message using tool calls. Call the \`submit\` tool at least once with the shortest summary possible that's <= ${length} words. The tool call will respond with the token count. Responses are ignored`;
|
||||
if(options?.system) system += '\n\n' + options.system;
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let done = false;
|
||||
const resp = await this.ask(text, {
|
||||
temperature: 0.3,
|
||||
...options,
|
||||
system,
|
||||
tools: [{
|
||||
name: 'submit',
|
||||
description: 'Submit summary',
|
||||
args: {summary: {type: 'string', description: 'Text summarization', required: true}},
|
||||
fn: (args) => {
|
||||
if(!args.summary) return 'No summary provided';
|
||||
const count = args.summary.split(' ').length;
|
||||
if(count > length) return `Too long: ${length} words`;
|
||||
done = true;
|
||||
resolve(args.summary || null);
|
||||
return `Saved: ${length} words`;
|
||||
}
|
||||
}, ...(options?.tools || [])],
|
||||
});
|
||||
if(!done) reject(`AI failed to create summary:\n${resp}`);
|
||||
});
|
||||
}
|
||||
|
||||
addModel(name: string, config: AnthropicConfig | OpenAiConfig, setDefault = false) {
|
||||
if(config.proto == 'anthropic') this.models[name] = new Anthropic(this.ai, config.token, name);
|
||||
else if(config.proto == 'openai') this.models[name] = new OpenAi(this.ai, config.host || null, config.token, name);
|
||||
if(setDefault || !this.defaultModel) this.defaultModel = name;
|
||||
}
|
||||
|
||||
removeModel(name: string) {
|
||||
delete this.models[name];
|
||||
if(this.defaultModel === name) {
|
||||
this.defaultModel = Object.keys(this.models)[0] ?? '';
|
||||
}
|
||||
}
|
||||
|
||||
setModels(models: {[model: string]: AnthropicConfig | OpenAiConfig}, replace = true) {
|
||||
if(replace) this.models = {};
|
||||
Object.entries(models).forEach(([model, config]) => {
|
||||
if(!this.defaultModel) this.defaultModel = model;
|
||||
if(config.proto == 'anthropic') this.models[model] = new Anthropic(this.ai, config.token, model);
|
||||
else if(config.proto == 'openai') this.models[model] = new OpenAi(this.ai, config.host || null, config.token, model);
|
||||
});
|
||||
this.defaultModel = Object.keys(this.models)[0] ?? '';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
642
src/memory.ts
Normal file
642
src/memory.ts
Normal file
@@ -0,0 +1,642 @@
|
||||
import {LLMRequest, LLMMessage} from './llm.ts';
|
||||
import {AiTool} from './tools.ts';
|
||||
import {KDPoint, KDTree} from './kd-tree.ts';
|
||||
|
||||
export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[] {
|
||||
const mems = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const nameSet = new Set(mems.map(m => m.name));
|
||||
const ghosts = new Set<string>();
|
||||
|
||||
const nodes: MemoryNode[] = mems.map(m => {
|
||||
const {links, backlinks} = extractMetadata(m.content);
|
||||
return {
|
||||
name: m.name,
|
||||
missing: false,
|
||||
links,
|
||||
backlinks,
|
||||
};
|
||||
});
|
||||
|
||||
for (const node of nodes) {
|
||||
for (const link of node.links) {
|
||||
if (!nameSet.has(link)) ghosts.add(link);
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
...nodes,
|
||||
...[...ghosts].map(name => ({
|
||||
name,
|
||||
missing: true,
|
||||
links: [],
|
||||
backlinks: nodes
|
||||
.filter(n => n.links.includes(name))
|
||||
.map(n => n.name),
|
||||
}))
|
||||
];
|
||||
}
|
||||
|
||||
export function renderMemoryGraph(nodes) {
|
||||
if (!nodes.length) return 'No memories yet.';
|
||||
|
||||
const groups = new Map();
|
||||
for (const node of nodes) {
|
||||
const [prefix, ...rest] = node.name.split('/');
|
||||
const group = rest.length ? prefix : 'Root';
|
||||
const label = rest.length ? rest.join('/') : node.name;
|
||||
if (!groups.has(group)) groups.set(group, []);
|
||||
groups.get(group).push({...node, label});
|
||||
}
|
||||
|
||||
const ghostCount = nodes.filter(n => n.missing).length;
|
||||
const lines = [`Memory Graph (${nodes.length} nodes, ${ghostCount} ghost${ghostCount === 1 ? '' : 's'})`, ''];
|
||||
|
||||
for (const group of [...groups.keys()].sort()) {
|
||||
const items = groups.get(group).sort((a, b) => a.label.localeCompare(b.label));
|
||||
lines.push(`${group}/`);
|
||||
items.forEach((n, i) => {
|
||||
const last = i === items.length - 1;
|
||||
const branch = last ? '└─' : '├─';
|
||||
const pad = last ? ' ' : '│ ';
|
||||
const tag = n.missing ? ' (ghost)' : '';
|
||||
lines.push(` ${branch} ${n.label}${tag}`);
|
||||
if (n.links.length) lines.push(` ${pad} → ${n.links.join(', ')}`);
|
||||
if (n.backlinks.length) lines.push(` ${pad} ← ${n.backlinks.join(', ')}`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n').trimEnd();
|
||||
}
|
||||
|
||||
export class MemoryCache {
|
||||
private tree: KDTree<MemoryRef>;
|
||||
public memories: Memory[];
|
||||
|
||||
get length() { return this.memories.length; }
|
||||
|
||||
constructor(memories: Memory[]) {
|
||||
this.memories = memories;
|
||||
this.tree = this.buildTree();
|
||||
}
|
||||
|
||||
private buildTree(): KDTree<MemoryRef> {
|
||||
const embedded = this.memories.filter(m => m.embedding?.length);
|
||||
if (!embedded.length) return new KDTree<MemoryRef>(0);
|
||||
|
||||
const dims = embedded[0].embedding.length;
|
||||
const points: KDPoint<MemoryRef>[] = embedded.map(m => ({
|
||||
vector: m.embedding,
|
||||
payload: {name: m.name, description: m.description},
|
||||
}));
|
||||
|
||||
return new KDTree<MemoryRef>(dims, 'cosine', points);
|
||||
}
|
||||
|
||||
search(query: number[], limit: number): MemoryRef[] {
|
||||
const results = this.tree.knn(query, limit);
|
||||
return results.map(r => r.point.payload);
|
||||
}
|
||||
|
||||
add(memory: Memory): void {
|
||||
this.memories.push(memory);
|
||||
this.rebuild();
|
||||
}
|
||||
|
||||
update(memory: Memory): void {
|
||||
const idx = this.memories.findIndex(m => m.name === memory.name);
|
||||
if (idx !== -1) {
|
||||
this.memories[idx] = memory;
|
||||
} else {
|
||||
this.memories.push(memory);
|
||||
}
|
||||
this.rebuild();
|
||||
}
|
||||
|
||||
remove(name: string): void {
|
||||
const idx = this.memories.findIndex(m => m.name === name);
|
||||
if (idx !== -1) {
|
||||
this.memories.splice(idx, 1);
|
||||
this.rebuild();
|
||||
}
|
||||
}
|
||||
|
||||
rebuild(): void {
|
||||
this.tree = this.buildTree();
|
||||
}
|
||||
}
|
||||
|
||||
export type MemoryOptions = {
|
||||
/** Memory object */
|
||||
memory: Memory[] | MemoryCache;
|
||||
/** Inject N memories into the system prompt */
|
||||
inject?: boolean;
|
||||
/** expose recall tool to LLM */
|
||||
tool?: boolean;
|
||||
/** Update memory on compression */
|
||||
update?: boolean;
|
||||
/** Max context size of memories to inject to each call (removed immediately after use) */
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
export type Memory = {
|
||||
name: string;
|
||||
description: string;
|
||||
content: string;
|
||||
embedding: number[];
|
||||
}
|
||||
|
||||
export type MemoryRef = {
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export type FactBucket = {
|
||||
subject: string;
|
||||
facts: string[];
|
||||
}
|
||||
|
||||
export type MemoryNode = {
|
||||
name: string;
|
||||
missing: boolean;
|
||||
links: string[];
|
||||
backlinks: string[];
|
||||
}
|
||||
|
||||
function extractLinks(content: string): string[] {
|
||||
if(!content) return [];
|
||||
const matches = content.matchAll(/\[\[([^\]]+)\]\]/g);
|
||||
return [...new Set([...matches].map(m => m[1].trim()))];
|
||||
}
|
||||
|
||||
export function extractMetadata(content: string): {links: string[], backlinks: string[]} {
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (!match) return {links: [], backlinks: []};
|
||||
|
||||
const fm = match[1];
|
||||
const getList = (key: string): string[] => {
|
||||
const m = fm.match(new RegExp(`^${key}:\\s*\\[(.*)\\]$`, 'm'));
|
||||
if (!m || !m[1].trim()) return [];
|
||||
return m[1].split(',').map(s => s.trim().replace(/^"|"$/g, '')).filter(Boolean);
|
||||
};
|
||||
|
||||
return {
|
||||
links: getList('links'),
|
||||
backlinks: getList('backlinks'),
|
||||
};
|
||||
}
|
||||
|
||||
function dedupeFacts(facts: string[]): string[] {
|
||||
const seen = new Map<string, string>();
|
||||
for (const f of facts) {
|
||||
const clean = f.trim();
|
||||
if (clean) seen.set(clean.toLowerCase(), clean);
|
||||
}
|
||||
return [...seen.values()];
|
||||
}
|
||||
|
||||
function cosineDistance(a: number[], b: number[]): number {
|
||||
let dot = 0, normA = 0, normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom === 0 ? 1 : 1 - dot / denom;
|
||||
}
|
||||
|
||||
function getWeekMonday(date: Date = new Date()): string {
|
||||
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
|
||||
const day = d.getUTCDay();
|
||||
const diff = day === 0 ? -6 : 1 - day;
|
||||
d.setUTCDate(d.getUTCDate() + diff);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function getWeekSunday(monday: string): string {
|
||||
const d = new Date(`${monday}T00:00:00Z`);
|
||||
d.setUTCDate(d.getUTCDate() + 6);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
export class MemoryManager {
|
||||
private recentlyTouched = new Map<string, number>();
|
||||
|
||||
private pendingMemorizations = new Map<string, {
|
||||
memories: Memory[] | MemoryCache,
|
||||
tempMemoryName: string,
|
||||
timestamp: number,
|
||||
}>();
|
||||
|
||||
private queues = new Map<string, {
|
||||
pending: string[],
|
||||
request: {abort?: () => void} | null,
|
||||
task: Promise<void>,
|
||||
}>();
|
||||
|
||||
tools = {
|
||||
read: (memories: Memory[] | MemoryCache): AiTool => ({
|
||||
name: 'memory_recall',
|
||||
description: 'Read the full content of a memory document',
|
||||
args: {
|
||||
name: {type: 'string', description: 'Exact memory name', required: true},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
const mems = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const mem = mems.find(m => m.name === args.name);
|
||||
if (!mem) return 'Document not found';
|
||||
this.touch(mem.name);
|
||||
return mem.content;
|
||||
},
|
||||
}),
|
||||
|
||||
forget: (memories: Memory[] | MemoryCache): AiTool => ({
|
||||
name: 'memory_forget',
|
||||
description: 'Permanently delete a memory document and clean up all references to it',
|
||||
args: {
|
||||
name: {type: 'string', description: 'Exact memory name to forget', required: true}
|
||||
},
|
||||
fn: (args: any) => {
|
||||
const result = this.forget(args.name, memories);
|
||||
return result ? `Forgotten: ${args.name}` : `Not found: ${args.name}`;
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
constructor(private llm: any) {}
|
||||
|
||||
static normalize(m?: Memory[] | MemoryCache | MemoryOptions) {
|
||||
if(!m) return null;
|
||||
const raw = m instanceof MemoryCache || Array.isArray(m);
|
||||
return raw ? {memory: <Memory[] | MemoryCache>m, inject: true, tool: true, update: true} : {inject: true, tool: true, update: true, ...m};
|
||||
}
|
||||
|
||||
private async createTempMemory(conversation: string): Promise<Memory> {
|
||||
const timestamp = Date.now();
|
||||
const content = `---
|
||||
name: _temp_${timestamp}
|
||||
description: Temporary memory - processing in background
|
||||
tags: [_temporary]
|
||||
links: []
|
||||
backlinks: []
|
||||
modified: ${new Date().toISOString()}
|
||||
---
|
||||
|
||||
# Recent Conversation (Processing)
|
||||
|
||||
${conversation}`;
|
||||
const [e] = await this.llm.embedding(content);
|
||||
return {
|
||||
name: `_temp_${timestamp}`,
|
||||
description: 'Temporary memory - processing in background',
|
||||
content,
|
||||
embedding: e?.embedding || [],
|
||||
};
|
||||
}
|
||||
|
||||
private applyHeader(content: string, header: string): string {
|
||||
return `${header}\n\n${this.stripHeader(content)}`;
|
||||
}
|
||||
|
||||
private async backgroundMemorization(conversation: string, memories: Memory[] | MemoryCache, options: LLMRequest, tempName: string): Promise<void> {
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const monday = getWeekMonday();
|
||||
const sunday = getWeekSunday(monday);
|
||||
const buckets = await this.factAgent(conversation, mem, options, monday);
|
||||
if(!buckets.length) return;
|
||||
const jobs = [...buckets].map(({subject, facts}) => {
|
||||
let node = mem.find(m => m.name === subject);
|
||||
if(!node) {
|
||||
node = {name: subject, description: '', content: '', embedding: [],};
|
||||
mem.push(node);
|
||||
}
|
||||
const week = subject.startsWith('Journal/') ? {monday, sunday} : undefined;
|
||||
return this.enqueue(node, facts, mem, options, tempName, week);
|
||||
});
|
||||
await Promise.all(jobs);
|
||||
}
|
||||
|
||||
private buildHeader(node: Memory, week?: {monday: string, sunday: string}, links: string[] = [], backlinks: string[] = []): string {
|
||||
const tags = node.name.split('/')[0]?.toLowerCase();
|
||||
const lines = [
|
||||
'---',
|
||||
`name: ${node.name}`,
|
||||
`description: ${node.description || ''}`,
|
||||
tags ? `tags: [${tags}]` : '',
|
||||
links.length ? `links: [${links.map(l => `"${l}"`).join(', ')}]` : 'links: []',
|
||||
backlinks.length ? `backlinks: [${backlinks.map(l => `"${l}"`).join(', ')}]` : 'backlinks: []',
|
||||
week ? `week: ${week.monday} – ${week.sunday}` : '',
|
||||
`modified: ${new Date().toISOString()}`,
|
||||
'---',
|
||||
].filter(Boolean);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] {
|
||||
const scored = memories
|
||||
.filter(m => m.embedding?.length)
|
||||
.map(m => ({
|
||||
ref: {name: m.name, description: m.description},
|
||||
distance: cosineDistance(query, m.embedding),
|
||||
}))
|
||||
.sort((a, b) => a.distance - b.distance)
|
||||
.slice(0, limit);
|
||||
return scored.map(s => s.ref);
|
||||
}
|
||||
|
||||
/**
|
||||
* Coalescing queue: if a doc is already compiling, abort the in-flight run, merge its
|
||||
* facts with the new ones and restart. Never blocks a pending update, never drops facts.
|
||||
*/
|
||||
private enqueue(node: Memory, facts: string[], memories: Memory[] | MemoryCache, options: LLMRequest, tempName: string, week?: {monday: string, sunday: string}): Promise<void> {
|
||||
const key = node.name;
|
||||
const existing = this.queues.get(key);
|
||||
if (existing) {
|
||||
existing.pending.push(...facts);
|
||||
existing.request?.abort?.();
|
||||
return existing.task;
|
||||
}
|
||||
|
||||
const entry: {pending: string[], request: {abort?: () => void} | null, task: Promise<void>} = {pending: [...facts], request: null, task: Promise.resolve()};
|
||||
this.queues.set(key, entry);
|
||||
const m = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
entry.task = (async () => {
|
||||
while (entry.pending.length) {
|
||||
const batch = dedupeFacts(entry.pending.splice(0, entry.pending.length));
|
||||
const written = await this.docAgent(node, batch, m, options, tempName, week, entry);
|
||||
if (!written) entry.pending.unshift(...batch);
|
||||
}
|
||||
})().finally(() => {
|
||||
this.queues.delete(key);
|
||||
if(!this.queues.size && memories instanceof MemoryCache) memories.rebuild();
|
||||
});
|
||||
return entry.task;
|
||||
}
|
||||
|
||||
private listNodes(memories: Memory[]): MemoryRef[] {
|
||||
return memories.map(m => ({name: m.name, description: m.description}));
|
||||
}
|
||||
|
||||
decay() {
|
||||
for(const [name, ttl] of this.recentlyTouched) {
|
||||
if(ttl <= 1) this.recentlyTouched.delete(name);
|
||||
else this.recentlyTouched.set(name, ttl - 1);
|
||||
}
|
||||
}
|
||||
|
||||
forget(name: string, memories: Memory[] | MemoryCache): boolean {
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const idx = mem.findIndex(m => m.name === name);
|
||||
if (idx === -1) return false;
|
||||
|
||||
for (const node of mem) {
|
||||
const {links, backlinks} = extractMetadata(node.content);
|
||||
const newBacklinks = backlinks.filter(b => b !== name);
|
||||
const newLinks = links.filter(l => l !== name);
|
||||
|
||||
if (newBacklinks.length !== backlinks.length || newLinks.length !== links.length) {
|
||||
node.content = this.updateFrontmatter(node.content, {
|
||||
links: newLinks,
|
||||
backlinks: newBacklinks,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
mem.splice(idx, 1);
|
||||
|
||||
if (memories instanceof MemoryCache) memories.rebuild();
|
||||
return true;
|
||||
}
|
||||
|
||||
getTouched(): string[] {
|
||||
return [...this.recentlyTouched.keys()];
|
||||
}
|
||||
|
||||
async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory[]> {
|
||||
const conversation = history
|
||||
.filter(h => h.role === 'user' || h.role === 'assistant')
|
||||
.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
|
||||
if(!conversation) return [];
|
||||
|
||||
const trackingId = `${Date.now()}_${Math.random()}`;
|
||||
const tempMemory = await this.createTempMemory(conversation);
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
mem.push(tempMemory);
|
||||
if (memories instanceof MemoryCache) memories.rebuild();
|
||||
this.pendingMemorizations.set(trackingId, {
|
||||
memories,
|
||||
tempMemoryName: tempMemory.name,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
try {
|
||||
await this.backgroundMemorization(conversation, memories, options, tempMemory.name);
|
||||
const finalMem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
return finalMem.filter(m => !m.name.startsWith('_temp_'));
|
||||
} finally {
|
||||
const pending = this.pendingMemorizations.get(trackingId);
|
||||
if (pending) {
|
||||
const cleanMem = pending.memories instanceof MemoryCache
|
||||
? pending.memories.memories
|
||||
: pending.memories;
|
||||
const idx = cleanMem.findIndex(m => m.name === pending.tempMemoryName);
|
||||
if (idx !== -1) cleanMem.splice(idx, 1);
|
||||
if (pending.memories instanceof MemoryCache) pending.memories.rebuild();
|
||||
}
|
||||
this.pendingMemorizations.delete(trackingId);
|
||||
}
|
||||
}
|
||||
|
||||
async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
|
||||
const mem: Memory[] = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
if (!mem.length) return [];
|
||||
|
||||
const [e] = await this.llm.embedding(query);
|
||||
if (!e) return [];
|
||||
|
||||
let vectorResults: MemoryRef[];
|
||||
if (memories instanceof MemoryCache) vectorResults = memories.search(e.embedding, limit);
|
||||
else vectorResults = this.cosineSearch(e.embedding, mem, limit);
|
||||
const found = new Set<string>(vectorResults.map(r => r.name));
|
||||
|
||||
if (graphDepth > 0) {
|
||||
const frontier = [...found];
|
||||
for (let depth = 0; depth < graphDepth; depth++) {
|
||||
const next: string[] = [];
|
||||
for (const name of frontier) {
|
||||
const node = mem.find(m => m.name === name);
|
||||
if (!node) continue;
|
||||
const {links} = extractMetadata(node.content);
|
||||
for (const link of links) {
|
||||
if (!found.has(link) && mem.find(m => m.name === link)) {
|
||||
found.add(link);
|
||||
next.push(link);
|
||||
}
|
||||
}
|
||||
}
|
||||
frontier.splice(0, frontier.length, ...next);
|
||||
if (!frontier.length) break;
|
||||
}
|
||||
}
|
||||
|
||||
const vectorOrder = vectorResults.map(r => r.name);
|
||||
const graphExpansions = [...found].filter(n => !vectorOrder.includes(n));
|
||||
const ordered = [...vectorOrder, ...graphExpansions];
|
||||
return ordered.map(n => mem.find(m => m.name === n)!).filter(Boolean);
|
||||
}
|
||||
|
||||
touch(name: string, ttl = 2) {
|
||||
this.recentlyTouched.set(name, ttl);
|
||||
}
|
||||
|
||||
private updateFrontmatter(content: string, updates: {links?: string[], backlinks?: string[]}): string {
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/);
|
||||
if (!match) return content;
|
||||
|
||||
const [, fm, body] = match;
|
||||
let newFm = fm;
|
||||
|
||||
if (updates.links !== undefined) {
|
||||
const linksList = updates.links.length ? `[${updates.links.map(l => `"${l}"`).join(', ')}]` : '[]';
|
||||
newFm = newFm.replace(/^links:.*$/m, `links: ${linksList}`);
|
||||
}
|
||||
|
||||
if (updates.backlinks !== undefined) {
|
||||
const backlinksList = updates.backlinks.length ? `[${updates.backlinks.map(l => `"${l}"`).join(', ')}]` : '[]';
|
||||
newFm = newFm.replace(/^backlinks:.*$/m, `backlinks: ${backlinksList}`);
|
||||
}
|
||||
|
||||
newFm = newFm.replace(/^modified:.*$/m, `modified: ${new Date().toISOString()}`);
|
||||
|
||||
return `---\n${newFm}\n---\n\n${body}`;
|
||||
}
|
||||
|
||||
private stripHeader(content: string): string {
|
||||
return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
|
||||
}
|
||||
|
||||
private async docAgent(node: Memory, facts: string[], memories: Memory[], options: LLMRequest, tempName: string, week: {monday: string, sunday: string} | undefined, entry: {request: {abort?: () => void} | null}): Promise<boolean> {
|
||||
const {links: oldLinks} = extractMetadata(node.content);
|
||||
const currentBody = this.stripHeader(node.content);
|
||||
let update;
|
||||
try {
|
||||
for(let i = 0; i < 3 && !update?.content; i++) {
|
||||
const request = this.llm.ask(`New Facts:\n${facts.map(f => `- ${f}`).join('\n')}`, {
|
||||
model: options.model,
|
||||
temperature: 0.3,
|
||||
schema: {
|
||||
description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true},
|
||||
content: {type: 'string', description: 'Rewritten document in markdown, without the frontmatter block', required: true},
|
||||
},
|
||||
system: `You are a knowledge base editor. Rewrite the current document below so it incorporates the new facts.
|
||||
|
||||
Formatting rules:
|
||||
- Use Obsidian-style markdown: # headings, **bold** to add emphasis, __italics__ for titles, terms, etc, bullet & numbered lists for grouped 1D data and tables for 2D data
|
||||
- Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]]
|
||||
- Create links for specific entities (person, place, project, program) and abstract concepts (quantum mechanics, entropy) but skip generics (car, red, dog)
|
||||
- Keep the document concise, factual, and human-readable
|
||||
- Resolve contradictions: the new facts always win — delete the outdated statement entirely, never keep both
|
||||
- Later facts in the list override earlier ones
|
||||
- Do not add frontmatter blocks, filler, preamble, or AI commentary
|
||||
${week ? '- This is a weekly journal entry.\n' : ''}
|
||||
All nodes:
|
||||
${this.listNodes(memories).map(n => n.name).join(', ') || 'none'}
|
||||
|
||||
Current document:
|
||||
\`\`\`markdown
|
||||
${currentBody}
|
||||
\`\`\``}
|
||||
);
|
||||
entry.request = request;
|
||||
update = await request;
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.name === 'AbortError') return false;
|
||||
throw err;
|
||||
} finally {
|
||||
entry.request = null;
|
||||
}
|
||||
|
||||
if(!update?.content) return false;
|
||||
const newLinks = extractLinks(update.content).filter(l => l !== node.name && l !== tempName);
|
||||
const newLinkSet = new Set(newLinks);
|
||||
const oldLinkSet = new Set(oldLinks);
|
||||
|
||||
for (const added of newLinkSet) {
|
||||
if (!oldLinkSet.has(added)) {
|
||||
const target = memories.find(m => m.name === added);
|
||||
if (target) {
|
||||
const {backlinks} = extractMetadata(target.content);
|
||||
if (!backlinks.includes(node.name)) {
|
||||
target.content = this.updateFrontmatter(target.content, {
|
||||
backlinks: [...backlinks, node.name],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const removed of oldLinkSet) {
|
||||
if (!newLinkSet.has(removed)) {
|
||||
const target = memories.find(m => m.name === removed);
|
||||
if (target) {
|
||||
const {backlinks} = extractMetadata(target.content);
|
||||
target.content = this.updateFrontmatter(target.content, {
|
||||
backlinks: backlinks.filter(b => b !== node.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const {backlinks} = extractMetadata(node.content);
|
||||
node.description = node.name !== 'Person/User' ? update.description : 'All information about the current user';
|
||||
node.content = this.applyHeader(update.content, this.buildHeader(node, week, newLinks, backlinks));
|
||||
const [e] = await this.llm.embedding(node.content);
|
||||
if(e) node.embedding = e.embedding;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async factAgent(conversation: string, memories: Memory[], options: LLMRequest, weekKey: string): Promise<FactBucket[]> {
|
||||
const buckets = new Map<string, string[]>();
|
||||
await this.llm.ask(conversation, {
|
||||
model: options.model,
|
||||
temperature: 0.2,
|
||||
system: `You are a fact extractor. Analyze this conversation and extract facts worth remembering long-term.
|
||||
|
||||
Rules:
|
||||
- ONLY extract current facts the USER explicitly stated about themselves, their work, or their projects
|
||||
- ONLY extract decisions that were MADE during this conversation
|
||||
- DO NOT extract anything the AI said, its capabilities, or meta-conversation about the AI
|
||||
- DO NOT extract greetings, pleasantries, or generic exchanges
|
||||
- DO NOT extract deltas or changes in facts; ONLY the end fact
|
||||
- If nothing worth remembering was said, do not call any tools
|
||||
|
||||
When extracting facts, you MUST also decide the exact destination path:
|
||||
- Use an existing node name if the facts clearly belong there
|
||||
- All information primary about the user should go under "People/User"
|
||||
- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide)
|
||||
- For journal entries, use "Journal"
|
||||
|
||||
Available nodes:
|
||||
- Journal
|
||||
${this.listNodes(memories).filter(n => !n.name.includes('_temp_') && !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`,
|
||||
tools: [{
|
||||
name: 'facts_extract',
|
||||
description: 'Submit facts with their destination',
|
||||
args: {
|
||||
destination: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
|
||||
facts: {type: 'string', description: 'Comma-separated facts', required: true},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
const subject = args.destination.trim().toLowerCase() === 'journal'
|
||||
? `Journal/${weekKey}` : args.destination.trim();
|
||||
const facts = buckets.get(subject) ?? [];
|
||||
facts.push(...dedupeFacts(String(args.facts).split(',')));
|
||||
buckets.set(subject, facts);
|
||||
return 'Recorded';
|
||||
},
|
||||
}],
|
||||
});
|
||||
return buckets.entries().toArray().map(([subject, facts]) => ({subject, facts}));
|
||||
}
|
||||
}
|
||||
118
src/open-ai.ts
118
src/open-ai.ts
@@ -3,6 +3,7 @@ import {findByProp, objectMap, JSONSanitize, JSONAttemptParse, clean} from '@zti
|
||||
import {AbortablePromise, Ai} from './ai.ts';
|
||||
import {LLMMessage, LLMRequest} from './llm.ts';
|
||||
import {LLMProvider} from './provider.ts';
|
||||
import {convertSchema} from './tools.ts';
|
||||
|
||||
export class OpenAi extends LLMProvider {
|
||||
client!: openAI;
|
||||
@@ -11,7 +12,7 @@ export class OpenAi extends LLMProvider {
|
||||
super();
|
||||
this.client = new openAI(clean({
|
||||
baseURL: host,
|
||||
apiKey: token
|
||||
apiKey: token || (host ? 'ignored' : undefined)
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -19,20 +20,22 @@ export class OpenAi extends LLMProvider {
|
||||
for(let i = 0; i < history.length; i++) {
|
||||
const h = history[i];
|
||||
if(h.role === 'assistant' && h.tool_calls) {
|
||||
const tools = h.tool_calls.map((tc: any) => ({
|
||||
const items: any[] = [];
|
||||
if(h.content) items.push({role: 'assistant', content: h.content, timestamp: h.timestamp});
|
||||
items.push(...h.tool_calls.map((tc: any) => ({
|
||||
role: 'tool',
|
||||
id: tc.id,
|
||||
name: tc.function.name,
|
||||
args: JSONAttemptParse(tc.function.arguments, {}),
|
||||
timestamp: h.timestamp
|
||||
}));
|
||||
history.splice(i, 1, ...tools);
|
||||
i += tools.length - 1;
|
||||
} else if(h.role === 'tool' && h.content) {
|
||||
})));
|
||||
history.splice(i, 1, ...items);
|
||||
i += items.length - 1;
|
||||
} else if(h.role === 'tool') {
|
||||
const record = history.find(h2 => h.tool_call_id == h2.id);
|
||||
if(record) {
|
||||
if(h.content.includes('"error":')) record.error = h.content;
|
||||
else record.content = h.content;
|
||||
if(h.content?.includes('"error":')) record.error = h.content;
|
||||
else record.content = h.content || '';
|
||||
}
|
||||
history.splice(i, 1);
|
||||
i--;
|
||||
@@ -50,32 +53,37 @@ export class OpenAi extends LLMProvider {
|
||||
content: null,
|
||||
tool_calls: [{ id: h.id, type: 'function', function: { name: h.name, arguments: JSON.stringify(h.args) } }],
|
||||
refusal: null,
|
||||
annotations: []
|
||||
annotations: [],
|
||||
timestamp: h.timestamp,
|
||||
}, {
|
||||
role: 'tool',
|
||||
tool_call_id: h.id,
|
||||
content: h.error || h.content
|
||||
content: h.error || h.content,
|
||||
timestamp: h.timestamp,
|
||||
});
|
||||
} else {
|
||||
const {timestamp, ...rest} = h;
|
||||
result.push(rest);
|
||||
result.push(h);
|
||||
}
|
||||
return result;
|
||||
}, [] as any[]);
|
||||
}
|
||||
|
||||
ask(message: string, options: LLMRequest = {}): AbortablePromise<string> {
|
||||
ask(message: string, options: LLMRequest = {}): AbortablePromise<string | any> {
|
||||
const controller = new AbortController();
|
||||
return Object.assign(new Promise<any>(async (res, rej) => {
|
||||
if(options.system && options.history?.[0]?.role != 'system') options.history?.splice(0, 0, {role: 'system', content: options.system, timestamp: Date.now()});
|
||||
let history = this.fromStandard([...options.history || [], {role: 'user', content: message, timestamp: Date.now()}]);
|
||||
const base = (options.history || []).filter(h => h.role !== 'system');
|
||||
let history = this.fromStandard([
|
||||
...(options.system ? [{role: <any>'system', content: options.system, timestamp: Date.now()}] : []),
|
||||
...base,
|
||||
{role: 'user', content: message, timestamp: Date.now()}
|
||||
]);
|
||||
const tools = options.tools || this.ai.options.llm?.tools || [];
|
||||
const requestParams: any = {
|
||||
model: options.model || this.model,
|
||||
messages: history,
|
||||
stream: !!options.stream,
|
||||
max_tokens: options.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
||||
temperature: options.temperature || this.ai.options.llm?.temperature || 0.7,
|
||||
max_completion_tokens: options.max_tokens || this.ai.options.llm?.max_tokens || undefined,
|
||||
temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
|
||||
tools: tools.map(t => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
@@ -90,17 +98,28 @@ export class OpenAi extends LLMProvider {
|
||||
}))
|
||||
};
|
||||
|
||||
let resp: any, isFirstMessage = true;
|
||||
if(options.schema) {
|
||||
const schema = convertSchema(options.schema);
|
||||
requestParams.response_format = {
|
||||
type: 'json_schema',
|
||||
json_schema: {
|
||||
name: 'response',
|
||||
strict: true,
|
||||
schema
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let resp: any, terminal = false;
|
||||
do {
|
||||
requestParams.messages = history.map(({timestamp, ...m}) => m);
|
||||
resp = await this.client.chat.completions.create(requestParams).catch(err => {
|
||||
err.message += `\n\nMessages:\n${JSON.stringify(history, null, 2)}`;
|
||||
throw err;
|
||||
});
|
||||
|
||||
if(options.stream) {
|
||||
if(!isFirstMessage) options.stream({text: '\n\n'});
|
||||
else isFirstMessage = false;
|
||||
resp.choices = [{message: {content: '', tool_calls: []}}];
|
||||
resp.choices = [{message: {role: 'assistant', content: '', tool_calls: [], timestamp: Date.now()}}];
|
||||
for await (const chunk of resp) {
|
||||
if(controller.signal.aborted) break;
|
||||
if(chunk.choices[0].delta.content) {
|
||||
@@ -108,36 +127,73 @@ export class OpenAi extends LLMProvider {
|
||||
options.stream({text: chunk.choices[0].delta.content});
|
||||
}
|
||||
if(chunk.choices[0].delta.tool_calls) {
|
||||
resp.choices[0].message.tool_calls = chunk.choices[0].delta.tool_calls;
|
||||
for(const deltaTC of chunk.choices[0].delta.tool_calls) {
|
||||
const existing = resp.choices[0].message.tool_calls.find(tc => tc.index === deltaTC.index);
|
||||
if(existing) {
|
||||
if(deltaTC.id) existing.id = deltaTC.id;
|
||||
if(deltaTC.type) existing.type = deltaTC.type;
|
||||
if(deltaTC.function) {
|
||||
if(!existing.function) existing.function = {};
|
||||
if(deltaTC.function.name) existing.function.name = deltaTC.function.name;
|
||||
if(deltaTC.function.arguments) existing.function.arguments = (existing.function.arguments || '') + deltaTC.function.arguments;
|
||||
}
|
||||
} else {
|
||||
resp.choices[0].message.tool_calls.push({
|
||||
index: deltaTC.index,
|
||||
id: deltaTC.id || '',
|
||||
type: deltaTC.type || 'function',
|
||||
function: {
|
||||
name: deltaTC.function?.name || '',
|
||||
arguments: deltaTC.function?.arguments || ''
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(resp.error) throw new Error(resp.error);
|
||||
const toolCalls = resp.choices[0].message.tool_calls || [];
|
||||
if(toolCalls.length && !controller.signal.aborted) {
|
||||
history.push(resp.choices[0].message);
|
||||
const results = await Promise.all(toolCalls.map(async (toolCall: any) => {
|
||||
const tool = tools?.find(findByProp('name', toolCall.function.name));
|
||||
if(options.stream) options.stream({tool: toolCall.function.name});
|
||||
if(!tool) return {role: 'tool', tool_call_id: toolCall.id, content: '{"error": "Tool not found"}'};
|
||||
if(!tool) return {role: 'tool', tool_call_id: toolCall.id, content: '{"error": "Tool not found"}', timestamp: Date.now()};
|
||||
try {
|
||||
const args = JSONAttemptParse(toolCall.function.arguments, {});
|
||||
const result = await tool.fn(args, options.stream, this.ai);
|
||||
return {role: 'tool', tool_call_id: toolCall.id, content: JSONSanitize(result)};
|
||||
// Wrap stream so a tool's `done` ends turn gracefully
|
||||
const toolStream = options.stream && ((chunk: any) => {
|
||||
if(chunk.done) { terminal = true; return; }
|
||||
options.stream!(chunk);
|
||||
});
|
||||
const result = await tool.fn(args, toolStream, this.ai, toolCall.id);
|
||||
return {role: 'tool', tool_call_id: toolCall.id, content: typeof result == 'object' ? JSONSanitize(result) : result, timestamp: Date.now()};
|
||||
} catch (err: any) {
|
||||
return {role: 'tool', tool_call_id: toolCall.id, content: JSONSanitize({error: err?.message || err?.toString() || 'Unknown'})};
|
||||
return {role: 'tool', tool_call_id: toolCall.id, content: JSONSanitize({error: err?.message || err?.toString() || 'Unknown'}), timestamp: Date.now()};
|
||||
}
|
||||
}));
|
||||
history.push(...results);
|
||||
requestParams.messages = history;
|
||||
}
|
||||
} while (!controller.signal.aborted && resp.choices?.[0]?.message?.tool_calls?.length);
|
||||
history.push({role: 'assistant', content: resp.choices[0].message.content || ''});
|
||||
history = this.toStandard(history);
|
||||
} while (!terminal && !controller.signal.aborted && resp.choices?.[0]?.message?.tool_calls?.length);
|
||||
|
||||
if(!terminal) {
|
||||
const textContent = resp.choices[0].message.content || '';
|
||||
history.push({role: 'assistant', content: textContent.trim(), timestamp: Date.now()});
|
||||
}
|
||||
history = this.toStandard(history);
|
||||
if(options.history) options.history.splice(0, options.history.length, ...history.filter(h => h.role !== 'system'));
|
||||
if(options.stream) options.stream({done: true});
|
||||
if(options.history) options.history.splice(0, options.history.length, ...history);
|
||||
res(history.at(-1)?.content);
|
||||
|
||||
const turnStart = history.map(h => h.role).lastIndexOf('user');
|
||||
const finalContent = history.slice(turnStart + 1).reduce((str, h) => {
|
||||
if(h.role === 'assistant') return str + (h.content || '');
|
||||
return str;
|
||||
}, '').trim();
|
||||
|
||||
res(options.schema ? JSONAttemptParse(finalContent, finalContent) : finalContent);
|
||||
}), {abort: () => controller.abort()});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {AbortablePromise} from './ai.ts';
|
||||
import {LLMMessage, LLMRequest} from './llm.ts';
|
||||
import {LLMRequest} from './llm.ts';
|
||||
|
||||
export abstract class LLMProvider {
|
||||
abstract ask(message: string, options: LLMRequest): AbortablePromise<string>;
|
||||
|
||||
763
src/tools.ts
763
src/tools.ts
@@ -1,9 +1,17 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
import {$, $Sync} from '@ztimson/node-utils';
|
||||
import {ASet, consoleInterceptor, Http, fn as Fn} from '@ztimson/utils';
|
||||
import {$Sync} from '@ztimson/node-utils';
|
||||
import {ASet, consoleInterceptor, Http, fn as Fn, decodeHtml, objectMap} from '@ztimson/utils';
|
||||
import * as os from 'node:os';
|
||||
import {Ai} from './ai.ts';
|
||||
import {LLMRequest} from './llm.ts';
|
||||
|
||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)';
|
||||
|
||||
const getShell = () => {
|
||||
if(os.platform() == 'win32') return 'cmd';
|
||||
return $Sync`echo $SHELL`?.split('/').pop() || 'bash';
|
||||
}
|
||||
|
||||
export type AiToolArg = {[key: string]: {
|
||||
/** Argument type */
|
||||
type: 'array' | 'boolean' | 'number' | 'object' | 'string',
|
||||
@@ -33,40 +41,103 @@ export type AiTool = {
|
||||
/** Tool arguments */
|
||||
args?: AiToolArg,
|
||||
/** Callback function */
|
||||
fn: (args: any, stream: LLMRequest['stream'], ai: Ai) => any | Promise<any>,
|
||||
fn: (args: any, stream: LLMRequest['stream'], ai: Ai, toolId?: string) => any | Promise<any>,
|
||||
};
|
||||
|
||||
export const CliTool: AiTool = {
|
||||
export function convertSchema(schema: any): any {
|
||||
if(!schema) return null;
|
||||
|
||||
const convertProp = (prop: any): any => {
|
||||
const converted: any = {
|
||||
type: prop.type || 'string',
|
||||
};
|
||||
|
||||
if(prop.description) converted.description = prop.description;
|
||||
if(prop.default !== undefined) converted.default = prop.default;
|
||||
if(prop.enum) converted.enum = prop.enum;
|
||||
if(prop.pattern) converted.pattern = prop.pattern;
|
||||
|
||||
// Handle array items
|
||||
if(prop.type === 'array' && prop.items) {
|
||||
converted.items = convertProp(prop.items);
|
||||
}
|
||||
|
||||
// Handle object properties
|
||||
if(prop.type === 'object' && prop.items) {
|
||||
converted.properties = objectMap(prop.items, (key, value) => convertProp(value));
|
||||
const required = Object.entries(prop.items).filter(([_, v]: any) => v.required).map(([k]) => k);
|
||||
if(required.length) converted.required = required;
|
||||
converted.additionalProperties = false;
|
||||
}
|
||||
|
||||
// Handle min/max based on type
|
||||
if(prop.min !== undefined) {
|
||||
if(prop.type === 'string' || prop.type === 'array') converted.minLength = prop.min;
|
||||
else converted.minimum = prop.min;
|
||||
}
|
||||
if(prop.max !== undefined) {
|
||||
if(prop.type === 'string' || prop.type === 'array') converted.maxLength = prop.max;
|
||||
else converted.maximum = prop.max;
|
||||
}
|
||||
|
||||
return converted;
|
||||
};
|
||||
|
||||
return {
|
||||
type: 'object',
|
||||
properties: objectMap(schema, (key, value) => convertProp(value)),
|
||||
required: Object.entries(schema).filter(([_, v]: any) => v.required).map(([k]) => k),
|
||||
additionalProperties: false
|
||||
};
|
||||
}
|
||||
|
||||
export const ExecCliTool: AiTool = {
|
||||
name: 'cli',
|
||||
description: 'Use the command line interface, returns any output',
|
||||
args: {command: {type: 'string', description: 'Command to run', required: true}},
|
||||
fn: (args: {command: string}) => $`${args.command}`
|
||||
fn: (args: {command: string}) => $Sync`${args.command}`
|
||||
}
|
||||
|
||||
export const DateTimeTool: AiTool = {
|
||||
name: 'get_datetime',
|
||||
description: 'Get current UTC date / time',
|
||||
args: {},
|
||||
fn: async () => new Date().toUTCString()
|
||||
export const ExecJSTool: AiTool = {
|
||||
name: 'exec_javascript',
|
||||
description: 'Execute commonjs javascript',
|
||||
args: {
|
||||
code: {type: 'string', description: 'CommonJS javascript', required: true}
|
||||
},
|
||||
fn: async (args: {code: string}) => {
|
||||
const c = consoleInterceptor(null);
|
||||
const resp = await Fn<any>({console: c}, args.code, true).catch((err: any) => c.output.error.push(err));
|
||||
return {...c.output, return: resp, stdout: undefined, stderr: undefined};
|
||||
}
|
||||
}
|
||||
|
||||
export const ExecPythonTool: AiTool = {
|
||||
name: 'exec_python',
|
||||
description: 'Execute commonjs javascript',
|
||||
args: {
|
||||
code: {type: 'string', description: 'CommonJS javascript', required: true}
|
||||
},
|
||||
fn: async (args: {code: string}) => ({result: $Sync`python -c "${args.code}"`})
|
||||
}
|
||||
|
||||
export const ExecTool: AiTool = {
|
||||
name: 'exec',
|
||||
description: 'Run code/scripts',
|
||||
args: {
|
||||
language: {type: 'string', description: 'Execution language', enum: ['cli', 'node', 'python'], required: true},
|
||||
language: {type: 'string', description: `Execution language (CLI: ${getShell()})`, enum: ['cli', 'node', 'python'], required: true},
|
||||
code: {type: 'string', description: 'Code to execute', required: true}
|
||||
},
|
||||
fn: async (args, stream, ai) => {
|
||||
try {
|
||||
switch(args.type) {
|
||||
case 'bash':
|
||||
return await CliTool.fn({command: args.code}, stream, ai);
|
||||
switch(args.language) {
|
||||
case 'cli':
|
||||
return await ExecCliTool.fn({command: args.code}, stream, ai);
|
||||
case 'node':
|
||||
return await JSTool.fn({code: args.code}, stream, ai);
|
||||
case 'python': {
|
||||
return await PythonTool.fn({code: args.code}, stream, ai);
|
||||
}
|
||||
return await ExecJSTool.fn({code: args.code}, stream, ai);
|
||||
case 'python':
|
||||
return await ExecPythonTool.fn({code: args.code}, stream, ai);
|
||||
default:
|
||||
throw new Error(`Unsupported language: ${args.language}`);
|
||||
}
|
||||
} catch(err: any) {
|
||||
return {error: err?.message || err.toString()};
|
||||
@@ -74,8 +145,483 @@ export const ExecTool: AiTool = {
|
||||
}
|
||||
}
|
||||
|
||||
export const FetchTool: AiTool = {
|
||||
name: 'fetch',
|
||||
export const FsDeleteTool = (whitelist: null | string[] = null): AiTool => {
|
||||
return {
|
||||
name: 'fs_delete',
|
||||
description: 'Delete a file or directory',
|
||||
args: {
|
||||
path: {type: 'string', description: 'Path to file or directory', required: true},
|
||||
recursive: {type: 'boolean', description: 'Delete all children', required: false}
|
||||
},
|
||||
fn: async ({path, recursive = false}) => {
|
||||
const {existsSync, rmSync} = await import('fs');
|
||||
const normalizePath = p => p.replace(/\\/g, '/');
|
||||
|
||||
path = normalizePath(path);
|
||||
if(whitelist && !whitelist.some(p => path.startsWith(p))) return {error: 'Permission denied'};
|
||||
if(!existsSync(path)) return {error: 'Path does not exist'};
|
||||
|
||||
rmSync(path, {recursive, force: true});
|
||||
return {success: true, path};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const FsMoveTool = (whitelist: null | string[] = null): AiTool => {
|
||||
return {
|
||||
name: 'fs_move',
|
||||
description: 'Move or rename a file or directory',
|
||||
args: {
|
||||
source: {type: 'string', description: 'Path to source file or directory', required: true},
|
||||
destination: {type: 'string', description: 'Path to destination file or directory', required: true}
|
||||
},
|
||||
fn: async ({source, destination}) => {
|
||||
const {existsSync, renameSync} = await import('fs');
|
||||
const normalizePath = p => p.replace(/\\/g, '/');
|
||||
|
||||
source = normalizePath(source);
|
||||
destination = normalizePath(destination);
|
||||
if(whitelist && !whitelist.some(p => source.startsWith(p) && destination.startsWith(p))) return {error: 'Permission denied'};
|
||||
|
||||
if(!existsSync(source)) return {error: 'Source path does not exist'};
|
||||
if(existsSync(destination)) return {error: 'Destination path already exists'};
|
||||
|
||||
renameSync(source, destination);
|
||||
return {success: true, source, destination};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const FsReadTool = (whitelist: null | string[] = null): AiTool => {
|
||||
return {
|
||||
name: 'fs_read',
|
||||
description: 'Read the contents of a provided path. Works with files and directories',
|
||||
args: {path: {type: 'string', description: 'Path to file or directory', required: true}},
|
||||
fn: async ({path}) => {
|
||||
const {existsSync, lstatSync, readdirSync, readFileSync} = await import('fs');
|
||||
const {join} = await import('path');
|
||||
const normalizePath = p => p.replace(/\\/g, '/');
|
||||
|
||||
path = normalizePath(path);
|
||||
if(whitelist && !whitelist.some(p => path.startsWith(p))) return {error: 'Permission denied'};
|
||||
|
||||
if(!existsSync(path)) return {error: 'Path does not exist'};
|
||||
const stats = lstatSync(path);
|
||||
if(stats.isDirectory()) {
|
||||
const children = readdirSync(path).map(name => {
|
||||
const childPath = normalizePath(join(path, name));
|
||||
const childStats = lstatSync(childPath);
|
||||
return {name, type: childStats.isDirectory() ? 'directory' : 'file', size: childStats.size};
|
||||
});
|
||||
return {type: 'directory', children};
|
||||
}
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
return {type: 'file', content};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const FsSearchTool = (whitelist: null | string[] = null): AiTool => {
|
||||
return {
|
||||
name: 'fs_search',
|
||||
description: 'Scan a directory for matching glob patterns (e.g. "**/*.js", "src/**/*.test.ts")',
|
||||
args: {
|
||||
pattern: {type: 'string', description: 'Glob pattern to match against paths', required: true},
|
||||
root: {type: 'string', description: 'Directory to search from', required: false, default: '.'}
|
||||
},
|
||||
fn: async ({pattern, root = '.'}) => {
|
||||
const {existsSync, lstatSync, readdirSync} = await import('fs');
|
||||
const {join, relative} = await import('path');
|
||||
const normalizePath = p => p.replace(/\\/g, '/');
|
||||
|
||||
root = normalizePath(root);
|
||||
if(!existsSync(root)) return {error: 'Root path does not exist'};
|
||||
if(!lstatSync(root).isDirectory()) return {error: 'Root path is not a directory'};
|
||||
|
||||
if(whitelist && !whitelist.some(p => root.startsWith(p))) return {error: 'Permission denied'};
|
||||
|
||||
const globToRegex = (glob) => {
|
||||
let re = '';
|
||||
for(let i = 0; i < glob.length; i++) {
|
||||
const c = glob[i];
|
||||
if(c === '*') {
|
||||
if(glob[i + 1] === '*') {
|
||||
const isSlash = glob[i + 2] === '/';
|
||||
re += '.*';
|
||||
i += isSlash ? 2 : 1;
|
||||
} else {
|
||||
re += '[^/]*';
|
||||
}
|
||||
} else if(c === '?') {
|
||||
re += '[^/]';
|
||||
} else if('.+^$(){}|[]\\'.includes(c)) {
|
||||
re += '\\' + c;
|
||||
} else {
|
||||
re += c;
|
||||
}
|
||||
}
|
||||
return new RegExp('^' + re + '$');
|
||||
};
|
||||
const regex = globToRegex(pattern);
|
||||
|
||||
const results: any = [];
|
||||
const walk = (dir) => {
|
||||
for(const name of readdirSync(dir)) {
|
||||
const fullPath = normalizePath(join(dir, name));
|
||||
const stats = lstatSync(fullPath);
|
||||
const relPath = normalizePath(relative(root, fullPath));
|
||||
if(regex.test(relPath)) {
|
||||
results.push({path: relPath, type: stats.isDirectory() ? 'directory' : 'file', size: stats.size});
|
||||
}
|
||||
if(stats.isDirectory()) walk(fullPath);
|
||||
}
|
||||
};
|
||||
walk(root);
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const FsWriteTool = (whitelist: null | string[] = null): AiTool => {
|
||||
return {
|
||||
name: 'fs_write',
|
||||
description: 'Create a directory, write content to a file or preform a find & replace',
|
||||
args: {
|
||||
path: {type: 'string', description: 'Path to file or directory', required: true},
|
||||
content: {type: 'string', description: 'Content to write or replace (Omit to create a directory)'},
|
||||
find: {type: 'string', description: 'Text or regex pattern to match (regex must match pattern: "/pattern/g")'}
|
||||
},
|
||||
fn: async ({path, content, find}) => {
|
||||
const {existsSync, mkdirSync, readFileSync, writeFileSync} = await import('fs');
|
||||
const {dirname} = await import('path');
|
||||
const normalizePath = p => p.replace(/\\/g, '/');
|
||||
|
||||
path = normalizePath(path);
|
||||
if(whitelist && !whitelist.some(p => path.startsWith(p))) return {error: 'Permission denied'};
|
||||
|
||||
if(content === undefined) {
|
||||
mkdirSync(path, {recursive: true});
|
||||
return {success: true, type: 'directory', path};
|
||||
}
|
||||
|
||||
const dir = normalizePath(dirname(path));
|
||||
if(!existsSync(dir)) mkdirSync(dir, {recursive: true});
|
||||
|
||||
if(find && existsSync(path)) {
|
||||
const existing = readFileSync(path, 'utf-8');
|
||||
const regexMatch = find.match(/^\/(.+)\/([gimuy]*)$/);
|
||||
const pattern = regexMatch ? new RegExp(regexMatch[1], regexMatch[2]) : find;
|
||||
|
||||
if(!existing.match(pattern)) return {error: 'Find pattern not found in file'};
|
||||
|
||||
const updated = existing.replace(pattern, content);
|
||||
writeFileSync(path, updated, 'utf-8');
|
||||
return {success: true, type: 'file', path, replaced: true, content: updated};
|
||||
}
|
||||
|
||||
writeFileSync(path, content, 'utf-8');
|
||||
return {success: true, type: 'file', path, content};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const GetPathsTool: AiTool = {
|
||||
name: 'get_paths',
|
||||
description: 'Get the current working directory, and paths to the users home directory',
|
||||
fn: async () => {
|
||||
return {
|
||||
home: os.homedir(),
|
||||
cwd: process.cwd()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const GetDatetimeTool: AiTool = {
|
||||
name: 'get_datetime',
|
||||
description: 'Get local/UTC timestamp',
|
||||
args: {
|
||||
timezone: {type: 'string', description: 'Which timezone to return, defaults to local', enum: ['local', 'utc'], default: 'local'}
|
||||
},
|
||||
fn: ({timezone}) => new Date()[timezone === 'local' ? 'toString' : 'toUTCString']()
|
||||
}
|
||||
|
||||
export const GetDevice: AiTool = {
|
||||
name: 'get_device',
|
||||
description: 'Get comprehensive system information including hostname, specs, load, storage, and network status',
|
||||
args: {},
|
||||
fn: async () => {
|
||||
const platform = os.platform();
|
||||
const hostname = os.hostname();
|
||||
|
||||
// CPU Info
|
||||
const cpus = os.cpus();
|
||||
const cpuModel = cpus[0].model;
|
||||
const cpuCores = cpus.length;
|
||||
|
||||
// Memory Info
|
||||
const totalMem: any = (os.totalmem() / 1024 / 1024 / 1024).toFixed(2);
|
||||
const freeMem: any = (os.freemem() / 1024 / 1024 / 1024).toFixed(2);
|
||||
const usedMem: any = (totalMem - freeMem).toFixed(2);
|
||||
const memUsage: any = ((usedMem / totalMem) * 100).toFixed(1);
|
||||
|
||||
// Load Average (not available on Windows)
|
||||
const loadAvg = platform === 'win32' ? ['N/A', 'N/A', 'N/A'] : os.loadavg().map(l => l.toFixed(2));
|
||||
|
||||
// Storage Usage
|
||||
let storage = {};
|
||||
if(platform === 'win32') {
|
||||
const ps = $Sync`powershell "Get-PSDrive C | Select-Object Used,Free | ConvertTo-Json"`.trim();
|
||||
const drive = JSON.parse(ps);
|
||||
const used: any = (drive.Used / 1024 / 1024 / 1024).toFixed(2);
|
||||
const free: any = (drive.Free / 1024 / 1024 / 1024).toFixed(2);
|
||||
const total: any = (parseFloat(used) + parseFloat(free)).toFixed(2);
|
||||
const usage: any = ((used / total) * 100).toFixed(1);
|
||||
storage = {
|
||||
filesystem: 'C:',
|
||||
size: `${total} GB`,
|
||||
used: `${used} GB`,
|
||||
available: `${free} GB`,
|
||||
usage: `${usage}%`
|
||||
};
|
||||
} else {
|
||||
const df = $Sync`df -h / | tail -1`.trim();
|
||||
const s = df.split(/\s+/);
|
||||
storage = {
|
||||
filesystem: s[0],
|
||||
size: s[1],
|
||||
used: s[2],
|
||||
available: s[3],
|
||||
usage: s[4]
|
||||
};
|
||||
}
|
||||
|
||||
// Network Status
|
||||
const interfaces = os.networkInterfaces();
|
||||
const activeIfaces = Object.entries(interfaces)
|
||||
.filter(([name]) => name !== 'lo' && !name.includes('Loopback'))
|
||||
.map(([name, addrs]) => {
|
||||
const ipv4 = addrs?.find(a => a.family === 'IPv4');
|
||||
return ipv4 ? {name, ip: ipv4.address} : null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
// Internet connectivity check
|
||||
let internet = false;
|
||||
try {
|
||||
if(platform === 'win32') {
|
||||
$Sync`powershell "Test-Connection -ComputerName 8.8.8.8 -Count 1 -Quiet"`;
|
||||
} else {
|
||||
$Sync`ping -c 1 -W 2 8.8.8.8 > /dev/null 2>&1`;
|
||||
}
|
||||
internet = true;
|
||||
} catch {}
|
||||
|
||||
// Uptime
|
||||
const uptime = os.uptime();
|
||||
const days = Math.floor(uptime / 86400);
|
||||
const hours = Math.floor((uptime % 86400) / 3600);
|
||||
const minutes = Math.floor((uptime % 3600) / 60);
|
||||
|
||||
return {
|
||||
hostname,
|
||||
cpu: {
|
||||
model: cpuModel,
|
||||
cores: cpuCores
|
||||
},
|
||||
memory: {
|
||||
total: `${totalMem} GB`,
|
||||
used: `${usedMem} GB`,
|
||||
free: `${freeMem} GB`,
|
||||
usage: `${memUsage}%`
|
||||
},
|
||||
load: {
|
||||
'1min': loadAvg[0],
|
||||
'5min': loadAvg[1],
|
||||
'15min': loadAvg[2]
|
||||
},
|
||||
storage,
|
||||
network: {
|
||||
interfaces: activeIfaces,
|
||||
internet: internet ? 'connected' : 'disconnected'
|
||||
},
|
||||
uptime: `${days}d ${hours}h ${minutes}m`,
|
||||
platform: `${os.type()} ${os.release()}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const GetWikipediaTool: AiTool = {
|
||||
name: 'get_wikipedia',
|
||||
description: 'Search Wikipedia for matching articles',
|
||||
args: {
|
||||
query: {type: 'string', description: 'Search term or article title', required: true},
|
||||
mode: {type: 'string', description: 'search - look for articles, summary - intro of first found article (default), full - complete first found article', enum: ['search', 'summary', 'full'], default: 'summary'},
|
||||
ua: {type: 'string', description: 'User Agent'},
|
||||
},
|
||||
fn: async ({query, mode, ua}) => {
|
||||
class WikipediaClient {
|
||||
useragent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)';
|
||||
|
||||
constructor(useragent: string) {
|
||||
this.useragent = useragent;
|
||||
}
|
||||
|
||||
async get(url) {
|
||||
const resp = await fetch(url, {headers: {'User-Agent': this.useragent}});
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
api(params) {
|
||||
const qs = new URLSearchParams({...params, format: 'json', utf8: '1'}).toString();
|
||||
return this.get(`https://en.wikipedia.org/w/api.php?${qs}`);
|
||||
}
|
||||
|
||||
clean(text) {
|
||||
const cutoffs = ['== See also ==', '== References ==', '== Bibliography ==', '== External links =='];
|
||||
for (const marker of cutoffs) {
|
||||
const idx = text.indexOf(marker);
|
||||
if (idx !== -1) text = text.slice(0, idx);
|
||||
}
|
||||
|
||||
return text
|
||||
.replace(/^={4}\s*(.+?)\s*={4}$/gm, '#### $1')
|
||||
.replace(/^={3}\s*(.+?)\s*={3}$/gm, '### $1')
|
||||
.replace(/^={2}\s*(.+?)\s*={2}$/gm, '## $1')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.replace(/ {2,}/g, ' ')
|
||||
.replace(/\[\d+]/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
async searchTitles(query: string, limit = 6) {
|
||||
const data = await this.api({action: 'query', list: 'search', srsearch: query, srlimit: limit, srprop: 'snippet'});
|
||||
return data.query?.search || [];
|
||||
}
|
||||
|
||||
async fetchExtract(title: string, introOnly = false) {
|
||||
const params: any = {action: 'query', prop: 'extracts', titles: title, explaintext: 1, redirects: 1};
|
||||
if(introOnly) params.exintro = 1;
|
||||
const data = await this.api(params);
|
||||
const page: any = Object.values(data.query?.pages || {})[0];
|
||||
return this.clean(page?.extract || '');
|
||||
}
|
||||
|
||||
pageUrl(title: string) {
|
||||
return `https://en.wikipedia.org/wiki/${encodeURIComponent(title.replace(/ /g, '_'))}`;
|
||||
}
|
||||
|
||||
stripHtml(text: string) {
|
||||
return text.replace(/<[^>]+>/g, '');
|
||||
}
|
||||
|
||||
async lookup(query: string, detail = 'summary') {
|
||||
const results = await this.searchTitles(query, 6);
|
||||
if(!results.length) return `❌ No Wikipedia articles found for "${query}"`;
|
||||
const title = results[0].title;
|
||||
const url = this.pageUrl(title);
|
||||
const introOnly = detail !== 'full';
|
||||
const content = await this.fetchExtract(title, introOnly);
|
||||
return `## ${title}\n🔗 ${url}\n\n${content}`;
|
||||
}
|
||||
|
||||
async search(query: string) {
|
||||
const results = await this.searchTitles(query, 8);
|
||||
if(!results.length) return `❌ No results for "${query}"`;
|
||||
const lines = [`### Search results for "${query}"\n`];
|
||||
for(let i = 0; i < results.length; i++) {
|
||||
const r = results[i];
|
||||
const snippet = this.stripHtml(r.snippet || '').trim();
|
||||
lines.push(`**${i + 1}. ${r.title}**\n${snippet}\n${this.pageUrl(r.title)}`);
|
||||
}
|
||||
return lines.join('\n\n');
|
||||
}
|
||||
}
|
||||
|
||||
const wiki = new WikipediaClient(ua);
|
||||
if(mode === 'search') return wiki.search(query);
|
||||
return wiki.lookup(query, mode || 'summary');
|
||||
}
|
||||
};
|
||||
|
||||
export const GeoCodeTool: AiTool = {
|
||||
name: 'geo_code',
|
||||
description: 'Converts coordinates to address OR vice versa',
|
||||
args: {
|
||||
query: {type: 'string', description: 'Search query - coordinates (lat,lon) or address string', required: true},
|
||||
},
|
||||
fn: async ({query}) => {
|
||||
const coordinates = /(-?\d+(?:\.\d+)?).*?,.*?(-?\d+(?:\.\d+)?)/.exec(query);
|
||||
if(coordinates) { // Geolocate
|
||||
const url = `https://nominatim.openstreetmap.org/reverse?format=json&lat=${encodeURIComponent(coordinates[1])}&lon=${encodeURIComponent(coordinates[2])}`;
|
||||
const response = await fetch(url, {headers: {'User-Agent': 'OpenSight/1.0', 'Accept-Language': 'en'}});
|
||||
const data = await response.json();
|
||||
if(data.display_name) return {address: data.display_name, mode: 'geolocate'};
|
||||
} else { // Geocode
|
||||
const url = `https://nominatim.openstreetmap.org/search?format=json&q=${encodeURIComponent(query)}`;
|
||||
const response = await fetch(url, {headers: {'User-Agent': 'OpenSight/1.0'}});
|
||||
const data = await response.json();
|
||||
if(data[0]) return {latitude: parseFloat(data[0].lat), longitude: parseFloat(data[0].lon), mode: 'geocode'};
|
||||
}
|
||||
return {error: 'Not found'};
|
||||
},
|
||||
}
|
||||
|
||||
export const GeoWeatherTool: AiTool = {
|
||||
name: 'geo_weather',
|
||||
description: 'Gets weather and air quality info for a location and time',
|
||||
args: {
|
||||
query: {type: 'string', description: 'Location - address or place name', required: true},
|
||||
day: {type: 'string', description: 'Date to retrieve (YYYY-MM-DD), defaults to today'},
|
||||
},
|
||||
fn: async ({query, day}) => {
|
||||
day = day || new Date().toISOString().slice(0, 10);
|
||||
|
||||
const geoUrl = `https://nominatim.openstreetmap.org/search?format=json&q=${encodeURIComponent(query)}`;
|
||||
const geoResponse = await fetch(geoUrl, {headers: {'User-Agent': 'OpenSight/1.0'}});
|
||||
const geoData = await geoResponse.json();
|
||||
if(!geoData[0]) return {error: 'Location not found'};
|
||||
|
||||
const lat = parseFloat(geoData[0].lat);
|
||||
const lon = parseFloat(geoData[0].lon);
|
||||
|
||||
const weatherUrl = `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lon}&start_date=${day}&end_date=${day}&daily=weathercode,temperature_2m_max,temperature_2m_min,apparent_temperature_max,apparent_temperature_min,precipitation_sum,precipitation_probability_max,windspeed_10m_max,winddirection_10m_dominant,uv_index_max,sunrise,sunset&timezone=auto`;
|
||||
const airUrl = `https://air-quality-api.open-meteo.com/v1/air-quality?latitude=${lat}&longitude=${lon}&start_date=${day}&end_date=${day}&hourly=us_aqi,european_aqi,pm10,pm2_5&timezone=auto`;
|
||||
|
||||
const [weatherResponse, airResponse] = await Promise.all([fetch(weatherUrl), fetch(airUrl)]);
|
||||
const weatherData = await weatherResponse.json();
|
||||
const airData = await airResponse.json();
|
||||
|
||||
const avg = arr => (arr && arr.length) ? arr.reduce((a, b) => a + b, 0) / arr.length : null;
|
||||
|
||||
return {
|
||||
location: geoData[0].display_name,
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
elevation: weatherData.elevation,
|
||||
date: day,
|
||||
weatherCode: weatherData.daily?.weathercode?.[0],
|
||||
tempMax: weatherData.daily?.temperature_2m_max?.[0],
|
||||
tempMin: weatherData.daily?.temperature_2m_min?.[0],
|
||||
feelsLikeMax: weatherData.daily?.apparent_temperature_max?.[0],
|
||||
feelsLikeMin: weatherData.daily?.apparent_temperature_min?.[0],
|
||||
precipitation: weatherData.daily?.precipitation_sum?.[0],
|
||||
precipitationChance: weatherData.daily?.precipitation_probability_max?.[0],
|
||||
windSpeedMax: weatherData.daily?.windspeed_10m_max?.[0],
|
||||
windDirection: weatherData.daily?.winddirection_10m_dominant?.[0],
|
||||
uvIndexMax: weatherData.daily?.uv_index_max?.[0],
|
||||
sunrise: weatherData.daily?.sunrise?.[0],
|
||||
sunset: weatherData.daily?.sunset?.[0],
|
||||
usAqi: avg(airData.hourly?.us_aqi),
|
||||
europeanAqi: avg(airData.hourly?.european_aqi),
|
||||
pm10: avg(airData.hourly?.pm10),
|
||||
pm2_5: avg(airData.hourly?.pm2_5),
|
||||
};
|
||||
},
|
||||
}
|
||||
|
||||
export const WebFetchTool: AiTool = {
|
||||
name: 'web_fetch',
|
||||
description: 'Make HTTP request to URL',
|
||||
args: {
|
||||
url: {type: 'string', description: 'URL to fetch', required: true},
|
||||
@@ -91,61 +637,160 @@ export const FetchTool: AiTool = {
|
||||
}) => new Http({url: args.url, headers: args.headers}).request({method: args.method || 'GET', body: args.body})
|
||||
}
|
||||
|
||||
export const JSTool: AiTool = {
|
||||
name: 'exec_javascript',
|
||||
description: 'Execute commonjs javascript',
|
||||
export const WebFlareSolverTool = (host: string) => {
|
||||
return {
|
||||
name: 'web_flaresolverr',
|
||||
description: 'Use a flaresolverr proxy to bypass cloudflare bot detection',
|
||||
args: {
|
||||
code: {type: 'string', description: 'CommonJS javascript', required: true}
|
||||
url: {type: 'string', description: 'URL to fetch', required: true},
|
||||
cmd: {type: 'string', description: 'Flaresolverr cmd', enum: ['request.get', 'request.post'], default: 'request.get'},
|
||||
maxTimeout: {type: 'number', description: 'Fetch time limit', default: 60_000},
|
||||
postData: {type: 'object', description: 'Data to send during request.post requests'},
|
||||
},
|
||||
fn: async (args: {code: string}) => {
|
||||
const console = consoleInterceptor(null);
|
||||
const resp = await Fn<any>({console}, args.code, true).catch((err: any) => console.output.error.push(err));
|
||||
return {...console.output, return: resp, stdout: undefined, stderr: undefined};
|
||||
fn: async ({url, cmd, maxTimeout, postData}) => {
|
||||
function toFormUrlEncoded(obj, prefix = '') {
|
||||
const pairs: any = [];
|
||||
for (const key in obj) {
|
||||
if (!obj.hasOwnProperty(key)) continue;
|
||||
|
||||
const value = obj[key];
|
||||
const encodedKey = prefix
|
||||
? `${prefix}[${encodeURIComponent(key)}]`
|
||||
: encodeURIComponent(key);
|
||||
|
||||
if (value === null || value === undefined) {
|
||||
pairs.push(`${encodedKey}=`);
|
||||
} else if (typeof value === 'object' && !Array.isArray(value)) {
|
||||
pairs.push(toFormUrlEncoded(value, encodedKey));
|
||||
} else if (Array.isArray(value)) {
|
||||
value.forEach(item => {
|
||||
pairs.push(`${encodedKey}[]=${encodeURIComponent(item)}`);
|
||||
});
|
||||
} else {
|
||||
pairs.push(`${encodedKey}=${encodeURIComponent(value)}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const PythonTool: AiTool = {
|
||||
name: 'exec_javascript',
|
||||
description: 'Execute commonjs javascript',
|
||||
args: {
|
||||
code: {type: 'string', description: 'CommonJS javascript', required: true}
|
||||
},
|
||||
fn: async (args: {code: string}) => ({result: $Sync`python -c "${args.code}"`})
|
||||
return pairs.join('&');
|
||||
}
|
||||
|
||||
export const ReadWebpageTool: AiTool = {
|
||||
name: 'read_webpage',
|
||||
description: 'Extract clean, structured content from a webpage. Use after web_search to read specific URLs',
|
||||
args: {
|
||||
url: {type: 'string', description: 'URL to extract content from', required: true},
|
||||
focus: {type: 'string', description: 'Optional: What aspect to focus on (e.g., "pricing", "features", "contact info")'}
|
||||
},
|
||||
fn: async (args: {url: string; focus?: string}) => {
|
||||
const html = await fetch(args.url, {headers: {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}})
|
||||
.then(r => r.text()).catch(err => {throw new Error(`Failed to fetch: ${err.message}`)});
|
||||
const res = await fetch(host + '/v1', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({cmd, url, maxTimeout, postData: postData ? toFormUrlEncoded(postData) : undefined}),
|
||||
});
|
||||
|
||||
if(!res.ok) throw new Error(`FlareSolverr HTTP error: ${res.status} ${res.statusText}`);
|
||||
const data = await res.json();
|
||||
if(data.status !== 'ok') throw new Error(`FlareSolverr error: ${data.message ?? data.status}`);
|
||||
return data.solution.response;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const WebReadTool: AiTool = {
|
||||
name: 'web_read',
|
||||
description: 'Extract clean content from webpages, or convert media/documents to accessible formats',
|
||||
args: {
|
||||
url: {type: 'string', description: 'URL to read', required: true},
|
||||
mimeRegex: {type: 'string', description: 'Optional regex to filter MIME types (e.g., "^image/", "text/")'}
|
||||
},
|
||||
fn: async (args: {url: string; mimeRegex?: string}) => {
|
||||
const ua = 'AiTools-Webpage/1.0';
|
||||
const maxSize = 10 * 1024 * 1024;
|
||||
|
||||
const response = await fetch(args.url, {
|
||||
headers: {
|
||||
'User-Agent': ua,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5'
|
||||
},
|
||||
redirect: 'follow'
|
||||
}).catch(err => {throw new Error(`Failed to fetch: ${err.message}`)});
|
||||
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
const mimeType = contentType.split(';')[0].trim().toLowerCase();
|
||||
|
||||
if(args.mimeRegex && !new RegExp(args.mimeRegex, 'i').test(mimeType)) {
|
||||
return `❌ MIME type rejected: ${mimeType} (filter: ${args.mimeRegex})`;
|
||||
}
|
||||
|
||||
if(mimeType.match(/^(image|audio|video)\//)) {
|
||||
const buffer = await response.arrayBuffer();
|
||||
if(buffer.byteLength > maxSize) {
|
||||
return `❌ File too large: ${(buffer.byteLength / 1024 / 1024).toFixed(1)}MB (max 10MB)\nType: ${mimeType}`;
|
||||
}
|
||||
const base64 = Buffer.from(buffer).toString('base64');
|
||||
return `## Media File\n**Type:** ${mimeType}\n**Size:** ${(buffer.byteLength / 1024).toFixed(1)}KB\n**Data URL:** \`data:${mimeType};base64,${base64.slice(0, 100)}...\``;
|
||||
}
|
||||
|
||||
if(mimeType.match(/^text\/(plain|csv|xml)/) || args.url.match(/\.(txt|csv|xml|md|yaml|yml)$/i)) {
|
||||
const text = await response.text();
|
||||
const truncated = text.length > 50000 ? text.slice(0, 50000) : text;
|
||||
return `## Text File\n**Type:** ${mimeType}\n**URL:** ${args.url}\n\n${truncated}`;
|
||||
}
|
||||
|
||||
if(mimeType.match(/application\/(json|xml|csv)/)) {
|
||||
const text = await response.text();
|
||||
const truncated = text.length > 50000 ? text.slice(0, 50000) : text;
|
||||
return `## Structured Data\n**Type:** ${mimeType}\n**URL:** ${args.url}\n\n\`\`\`\n${truncated}\n\`\`\``;
|
||||
}
|
||||
|
||||
if(mimeType === 'application/pdf' || (mimeType.startsWith('application/') && !mimeType.includes('html'))) {
|
||||
const buffer = await response.arrayBuffer();
|
||||
if(buffer.byteLength > maxSize) {
|
||||
return `❌ File too large: ${(buffer.byteLength / 1024 / 1024).toFixed(1)}MB (max 10MB)\nType: ${mimeType}`;
|
||||
}
|
||||
const base64 = Buffer.from(buffer).toString('base64');
|
||||
return `## Binary File\n**Type:** ${mimeType}\n**Size:** ${(buffer.byteLength / 1024).toFixed(1)}KB\n**Data URL:** \`data:${mimeType};base64,${base64.slice(0, 100)}...\``;
|
||||
}
|
||||
|
||||
// HTML
|
||||
const html = await response.text();
|
||||
const $ = cheerio.load(html);
|
||||
$('script, style, nav, footer, header, aside, iframe, noscript, [role="navigation"], [role="banner"], .ad, .ads, .cookie, .popup').remove();
|
||||
const metadata = {
|
||||
title: $('meta[property="og:title"]').attr('content') || $('title').text() || '',
|
||||
description: $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '',
|
||||
};
|
||||
|
||||
$('script, style, nav, footer, header, aside, iframe, noscript, svg').remove();
|
||||
$('[role="navigation"], [role="banner"], [role="complementary"]').remove();
|
||||
$('[aria-hidden="true"], [hidden], .visually-hidden, .sr-only, .screen-reader-text').remove();
|
||||
$('.ad, .ads, .advertisement, .cookie, .popup, .modal, .sidebar, .related, .comments, .social-share').remove();
|
||||
$('button, [class*="share"], [class*="follow"], [class*="social"]').remove();
|
||||
const title = $('meta[property="og:title"]').attr('content') || $('title').text().trim() || '';
|
||||
const description = $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '';
|
||||
const author = $('meta[name="author"]').attr('content') || '';
|
||||
let content = '';
|
||||
const contentSelectors = ['article', 'main', '[role="main"]', '.content', '.post', '.entry', 'body'];
|
||||
for (const selector of contentSelectors) {
|
||||
const el = $(selector).first();
|
||||
const selectors = ['article', 'main', '[role="main"]', '.content', '.post-content', '.entry-content', '.article-content'];
|
||||
for(const sel of selectors) {
|
||||
const el = $(sel).first();
|
||||
if(el.length && el.text().trim().length > 200) {
|
||||
content = el.text();
|
||||
const paragraphs: string[] = [];
|
||||
el.find('p').each((_, p) => {
|
||||
const text = $(p).text().trim();
|
||||
if(text.length > 80) paragraphs.push(text);
|
||||
});
|
||||
if(paragraphs.length > 2) {
|
||||
content = paragraphs.join('\n\n');
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!content) content = $('body').text();
|
||||
content = content.replace(/\s+/g, ' ').trim().slice(0, 8000);
|
||||
}
|
||||
|
||||
return {url: args.url, title: metadata.title.trim(), description: metadata.description.trim(), content, focus: args.focus};
|
||||
if(!content) {
|
||||
const paragraphs: string[] = [];
|
||||
$('body p').each((_, p) => {
|
||||
const text = $(p).text().trim();
|
||||
if(text.length > 80) paragraphs.push(text);
|
||||
});
|
||||
content = paragraphs.slice(0, 30).join('\n\n');
|
||||
}
|
||||
|
||||
// Decode escaped newlines and clean
|
||||
const parts = [`## ${title || 'Webpage'}`];
|
||||
if(description) parts.push(`_${description}_`);
|
||||
if(author) parts.push(`👤 ${author}`);
|
||||
parts.push(`🔗 ${args.url}\n`);
|
||||
parts.push(content);
|
||||
return decodeHtml(parts.join('\n\n').replaceAll(/\n{3,}/g, '\n\n'));
|
||||
}
|
||||
};
|
||||
|
||||
export const WebSearchTool: AiTool = {
|
||||
name: 'web_search',
|
||||
@@ -159,7 +804,7 @@ export const WebSearchTool: AiTool = {
|
||||
length: number;
|
||||
}) => {
|
||||
const html = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(args.query)}`, {
|
||||
headers: {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", "Accept-Language": "en-US,en;q=0.9"}
|
||||
headers: {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"}
|
||||
}).then(resp => resp.text());
|
||||
let match, regex = /<a .*?href="(.+?)".+?<\/a>/g;
|
||||
const results = new ASet<string>();
|
||||
|
||||
@@ -12,12 +12,31 @@ export class Vision {
|
||||
*/
|
||||
ocr(path: string): AbortablePromise<string | null> {
|
||||
let worker: any;
|
||||
const p = new Promise<string | null>(async res => {
|
||||
let reject: (err: any) => void;
|
||||
|
||||
const handler = (err: Error) => {
|
||||
if(err.stack?.includes('tesseract.js')) {
|
||||
process.off('uncaughtException', handler);
|
||||
reject?.(err);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
};
|
||||
process.on('uncaughtException', handler);
|
||||
|
||||
const p = (async () => {
|
||||
worker = await createWorker(this.ai.options.ocr || 'eng', 2, {cachePath: this.ai.options.path});
|
||||
const {data} = await worker.recognize(path);
|
||||
await worker.terminate();
|
||||
res(data.text.trim() || null);
|
||||
return await new Promise<string | null>((res, rej) => {
|
||||
reject = rej;
|
||||
worker.recognize(path)
|
||||
.then(({data}: any) => res(data.text.trim() || null))
|
||||
.catch(rej);
|
||||
});
|
||||
})().finally(() => {
|
||||
process.off('uncaughtException', handler);
|
||||
worker?.terminate();
|
||||
});
|
||||
|
||||
return Object.assign(p, {abort: () => worker?.terminate()});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
"target": "ESNext",
|
||||
"useDefineForClassFields": true,
|
||||
"module": "ESNext",
|
||||
"lib": ["ESNext"],
|
||||
"lib": [
|
||||
"ESNext",
|
||||
"dom"
|
||||
],
|
||||
"skipLibCheck": true,
|
||||
|
||||
/* Bundler mode */
|
||||
@@ -15,6 +18,7 @@
|
||||
"noEmit": true,
|
||||
|
||||
/* Linting */
|
||||
"strict": true
|
||||
"strict": true,
|
||||
"noImplicitAny": false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import {defineConfig} from 'vite';
|
||||
import dts from 'vite-plugin-dts';
|
||||
import {resolve} from 'path';
|
||||
|
||||
export default defineConfig({
|
||||
build: {
|
||||
lib: {
|
||||
entry: {
|
||||
asr: './src/asr.ts',
|
||||
index: './src/index.ts',
|
||||
embedder: './src/embedder.ts',
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user