Proper error handling for OCR

main.mjs

@@ -17,5 +17,9 @@ const skills = [{
 }];
 
 const history = [], memory = [];
-console.log(await ai.language.ask('Can you tell me how to use momentum?', {history, skills}));
+await ai.language.ask('My favorite color is red', {history, memory});
-console.log(history, memory);
+await ai.language.updateMemory(history, memory);
+
+history.splice(0, history.length);
+console.log(await ai.language.ask('Whats my favorite color?', {history, memory}));
+console.log(history);

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.0.0",
+	"version": "1.0.5",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/index.ts

@@ -2,6 +2,7 @@ export * from './ai';
 export * from './antrhopic';
 export * from './audio';
 export * from './llm';
+export * from './memory';
 export * from './open-ai';
 export * from './provider';
 export * from './tools';

src/llm.ts

@@ -171,8 +171,8 @@ class LLM {
 		let abort = () => {};
 		return Object.assign(new Promise<string>(async res => {
 			let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
-			const prompts: string[] = [options.system || this.ai.options.llm?.system || ''];
+			const prompts: string[] = [];
-			if(!options.history) options.history = [];
+			let history = options.history || [];
 
 			// MCP
 			const mcp = options.mcp || this.ai.options?.llm?.mcp;
@@ -192,30 +192,33 @@ class LLM {
 
 			// Memory
 			if(options.memory) {
-				const relevant = await this.memoryManager.recollect(message, options.memory);
+				const relevant = await this.memoryManager.recollect(message, options.memory, 1);
-				if(relevant.length) {
+				prompts.unshift(`You have access to the following memory files:
-					const context = relevant.map(m => `### ${m.name}\n${m.content}`).join('\n\n');
+${options.memory.map(m => `- ${m.name}: ${m.description}`).join('\n')}
-					options.history.push({
+${relevant.length ? `
-						id: 'auto_recall_' + Math.random().toString(), role: 'tool', name: 'recall', args: {},
+The closest memory has been added primitively:
-						content: `Knowledge Documents:\n\n${context}`
+\`\`\`
-					});
+Name: ${relevant[0].name}
-				}
+Description: ${relevant[0].description}
-				prompts.unshift('You have access to a knowledge base. Relevant documents are injected automatically before each message. Use this knowledge to inform your responses.');
+${relevant[0].content}
+\`\`\`
+`: ''}`.trim());
+				tools.push(this.memoryManager.tools.read(<Memory[]>options.memory));
 			}
 
+			prompts.unshift(options.system || this.ai.options.llm?.system || '');
 			const resp = await this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
 
 			// Trim memory injections from history
 			if(options.memory) {
-				options.history.splice(0, options.history.length, ...options.history.filter(h =>
+				history.splice(0, history.length, ...history.filter(h => h.role !== 'tool' || h.name !== 'recall'));
-					h.role !== 'tool' || h.name !== 'recall'));
 			}
 
 			// Auto-memorize before compressing
-			if(options.compress) {
+			if(options.compress && this.estimateTokens(history) >= options.compress.max) {
-				if(options.memory) await this.memoryManager.memorize(options.history, options.memory, options);
+				if(options.memory) await this.memoryManager.memorize(history, options.memory, options);
-				const compressed = await this.compressHistory(options.history, options.compress.max, options.compress.min, options);
+				const compressed = await this.compressHistory(history, options.compress.max, options.compress.min, options);
-				options.history.splice(0, options.history.length, ...compressed);
+				if(options.history) options.history.splice(0, options.history.length, ...compressed);
 			}
 
 			return res(resp);
@@ -458,6 +461,31 @@ class LLM {
 			if(!done) reject(`AI failed to create summary:\n${resp}`);
 		});
 	}
+
+	addModel(name: string, config: AnthropicConfig | OllamaConfig | OpenAiConfig, setDefault = false) {
+		if(config.proto == 'anthropic') this.models[name] = new Anthropic(this.ai, config.token, name);
+		else if(config.proto == 'ollama') this.models[name] = new OpenAi(this.ai, config.host, 'not-needed', name);
+		else if(config.proto == 'openai') this.models[name] = new OpenAi(this.ai, config.host || null, config.token, name);
+		if(setDefault || !this.defaultModel) this.defaultModel = name;
+	}
+
+	removeModel(name: string) {
+		delete this.models[name];
+		if(this.defaultModel === name) {
+			this.defaultModel = Object.keys(this.models)[0] ?? '';
+		}
+	}
+
+	setModels(models: {[model: string]: AnthropicConfig | OllamaConfig | OpenAiConfig}, replace = true) {
+		if(replace) this.models = {};
+		Object.entries(models).forEach(([model, config]) => {
+			if(!this.defaultModel) this.defaultModel = model;
+			if(config.proto == 'anthropic') this.models[model] = new Anthropic(this.ai, config.token, model);
+			else if(config.proto == 'ollama') this.models[model] = new OpenAi(this.ai, config.host, 'not-needed', model);
+			else if(config.proto == 'openai') this.models[model] = new OpenAi(this.ai, config.host || null, config.token, model);
+		});
+		this.defaultModel = Object.keys(this.models)[0] ?? '';
+	}
 }
 
 export default LLM;

src/memory.ts

@@ -1,5 +1,6 @@
 // memory.ts
 import {LLMRequest, LLMMessage} from './llm.ts';
+import {AiTool} from './tools.ts';
 
 /** Background information the AI will be fed as a knowledge document */
 export type Memory = {
@@ -25,9 +26,9 @@ export type MemoryCollection = {
 export class MemoryManager {
 
 	tools = {
-		edit: (memory: Memory) => ({
+		edit: (memory: Memory): AiTool => ({
-			name: 'edit',
+			name: 'edit_memory',
-			description: 'Edit a memory. Omit start/end to append. Pass start only to replace from that line on. Pass start+end to replace a specific range. start=0 replaces the whole document.',
+			description: 'Edit a memory. Omit start/end to append. Pass start only to replace from that line on (Note line 0 = first line of content / line AFTER description). Pass start+end to replace a specific range. start=0 replaces the whole document. Returns updated document',
 			args: {
 				content: {type: 'string', description: 'New content', required: true},
 				start: {type: 'number', description: 'First line to replace (0-indexed, inclusive). Omit to append.'},
@@ -40,15 +41,15 @@ export class MemoryManager {
 				else if(args.end === undefined) lines.splice(args.start, lines.length - args.start, ...newLines);
 				else lines.splice(args.start, args.end - args.start + 1, ...newLines);
 				memory.content = lines.join('\n');
-				return `Updated memory:\n${memory.content}`;
+				return memory.content;
 			}
 		}),
-		extract: (pools: MemoryCollection[]) => ({
+		extract: (pools: MemoryCollection[]): AiTool => ({
-			name: 'extract',
+			name: 'extract_facts',
 			description: 'Extract a list of facts to group into a single memory',
 			args: {
 				name: {type: 'string', description: 'Exact name of an existing memory, or a new name if none fits ([pro]nouns only)', required: true},
-				description: {type: 'string', description: 'One sentence description of the memory subject, only required if new'},
+				description: {type: 'string', description: 'One sentence description of the memory subject', required: true},
 				facts: {type: 'string', description: 'Comma separated list of extracted facts', required: true},
 			},
 			fn: (args: any) => {
@@ -59,8 +60,8 @@ export class MemoryManager {
 				});
 				return 'Success';
 			}}),
-		read: (memories: Memory[]) => ({
+		read: (memories: Memory[]): AiTool => ({
-			name: 'read',
+			name: 'read_memory',
 			description: 'Read entire memory',
 			args: {
 				name: {type: 'string', description: 'Exact memory name', required: true},
@@ -94,9 +95,9 @@ Rules:
 - ONLY extract decisions that were MADE during this conversation
 - DO NOT extract anything the AI said, its name, capabilities, or how it introduced itself
 - DO NOT extract greetings, pleasantries or generic exchanges
-- If nothing worth remembering was said, call NO tools
+- If nothing worth remembering was said, dont do anything, skip calling tools
 
-For each fact decide whether it belongs in an existing document or needs a new one, then call the \`extract\` tool.
+For each fact decide whether it belongs in an existing document or needs a new one, then call the \`extract_facts\` tool.
 
 Existing documents:\n${existingDocs || 'None yet.'}`,
 			tools: [this.tools.extract(pools)]
@@ -120,18 +121,17 @@ Existing documents:\n${existingDocs || 'None yet.'}`,
 			{
 				model: this.model || options.model,
 				temperature: 0.2,
-				system: `You are a document editor. Merge the users list of facts into the following document using the \`edit\` tool; call it as many times as necessary.
+				system: `You are a document editor. Merge the users list of facts into the following document using the \`edit_memory\` tool; call it as many times as necessary:
-
+\`\`\`
-Name: ${mem.name}
+${mem.content}
-Description: ${mem.description}
+\`\`\``,
-${mem.content}`,
 				tools: [this.tools.edit(mem)]
 			}
 		);
 
 		if(isNew || mem.description !== existing?.description) {
-			const [e] = await this.llm.embedding(mem.description);
+			const e = await this.llm.embedding(mem.description);
-			mem.embedding = e.embedding;
+			mem.embedding = e?.[0]?.embedding;
 		}
 
 		if(isNew) memories.push(mem);

src/vision.ts

@@ -12,12 +12,16 @@ export class Vision {
 	 */
 	ocr(path: string): AbortablePromise<string | null> {
 		let worker: any;
-		const p = new Promise<string | null>(async res => {
+		const p = (async () => {
 			worker = await createWorker(this.ai.options.ocr || 'eng', 2, {cachePath: this.ai.options.path});
-			const {data} = await worker.recognize(path);
+			worker.setParameters({}).catch(() => {}); // force error handler attachment
-			await worker.terminate();
+			return await new Promise<string | null>((res, rej) => {
-			res(data.text.trim() || null);
+				worker.on?.('error', rej); // catch worker-level throws
-		}).finally(() => worker?.terminate());
+				worker.recognize(path)
+					.then(({data}: any) => res(data.text.trim() || null))
+					.catch(rej);
+			});
+		})().finally(() => worker?.terminate());
 		return Object.assign(p, {abort: () => worker?.terminate()});
 	}
 }