Handle anthropic multiple responses better.

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "0.1.13",
+	"version": "0.1.16",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/ai.ts

@@ -1,9 +1,9 @@
-import {$} from '@ztimson/node-utils';
 import {createWorker} from 'tesseract.js';
 import {LLM, LLMOptions} from './llm';
 import fs from 'node:fs/promises';
 import Path from 'node:path';
 import * as tf from '@tensorflow/tfjs';
+import {spawn} from 'node:child_process';
 
 export type AiOptions = LLMOptions & {
 	whisper?: {
@@ -13,8 +13,6 @@ export type AiOptions = LLMOptions & {
 		model: string;
 		/** Path to models */
 		path: string;
-		/** Path to storage location for temporary files */
-		temp?: string;
 	}
 }
 
@@ -29,7 +27,6 @@ export class Ai {
 		this.llm = new LLM(this, options);
 		if(this.options.whisper?.binary) {
 			this.whisperModel = this.options.whisper?.model.endsWith('.bin') ? this.options.whisper?.model : this.options.whisper?.model + '.bin';
-			console.log('constructor: ' + this.options.whisper.model + ' -> ' + this.whisperModel);
 			this.downloadAsrModel();
 		}
 	}
@@ -40,16 +37,23 @@ export class Ai {
 	 * @param model Whisper model
 	 * @returns {Promise<any>} Extracted text
 	 */
-	async asr(path: string, model: string = this.whisperModel): Promise<string | null> {
+	asr(path: string, model: string = this.whisperModel): {abort: () => void, response: Promise<string | null>} {
 		if(!this.options.whisper?.binary) throw new Error('Whisper not configured');
-		const m = await this.downloadAsrModel(model);
-		const name = Math.random().toString(36).substring(2, 10) + '-' + path.split('/').pop() + '.txt';
-		const output = Path.join(this.options.whisper.temp || '/tmp', name);
-		console.log('ASR: ' + this.options.whisper.model + ' -> ' + this.whisperModel);
-		console.log(`rm -f ${output} && ${this.options.whisper.binary} -nt -np -m ${m} -f ${path} -otxt -of ${output}`);
-		await $`rm -f ${output} && ${this.options.whisper.binary} -nt -np -m ${m} -f ${path} -otxt -of ${output}`;
-		return fs.readFile(output, 'utf-8').then(text => text?.trim() || null)
-			.finally(() => fs.rm(output, {force: true}).catch(() => {}));
+		let abort: any = () => {};
+		const response = new Promise<string | null>((resolve, reject) => {
+			this.downloadAsrModel(model).then(m => {
+				let output = '';
+				const proc = spawn(<string>this.options.whisper?.binary, ['-nt', '-np', '-m', m, '-f', path], {stdio: ['ignore', 'pipe', 'ignore']});
+				abort = () => proc.kill('SIGTERM');
+				proc.on('error', (err: Error) => reject(err));
+				proc.stdout.on('data', (data: Buffer) => output += data.toString());
+				proc.on('close', (code: number) => {
+					if(code === 0) resolve(output.trim() || null);
+					else reject(new Error(`Exit code ${code}`));
+				});
+			});
+		});
+		return {response, abort};
 	}
 
 	/**
@@ -62,11 +66,7 @@ export class Ai {
 		if(!this.options.whisper?.binary) throw new Error('Whisper not configured');
 		if(!model.endsWith('.bin')) model += '.bin';
 		const p = Path.join(this.options.whisper.path, model);
-		console.log('Download: ' + p);
-		if(await fs.stat(p).then(() => true).catch(() => false)) {
-			console.log('Exists!');
-			return p;
-		}
+		if(await fs.stat(p).then(() => true).catch(() => false)) return p;
 		if(!!this.downloads[model]) return this.downloads[model];
 		this.downloads[model] = fetch(`https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${model}`)
 			.then(resp => resp.arrayBuffer())

src/antrhopic.ts

@@ -13,24 +13,29 @@ export class Anthropic extends LLMProvider {
 	}
 
 	private toStandard(history: any[]): LLMMessage[] {
+		const merged: any[] = [];
 		for(let i = 0; i < history.length; i++) {
-			const orgI = i;
-			if(typeof history[orgI].content != 'string') {
-				if(history[orgI].role == 'assistant') {
-					history[orgI].content.filter((c: any) => c.type =='tool_use').forEach((c: any) => {
-						i++;
-						history.splice(i, 0, {role: 'tool', id: c.id, name: c.name, args: c.input});
+			const msg = history[i];
+			if(typeof msg.content != 'string') {
+				if(msg.role == 'assistant') {
+					msg.content.filter((c: any) => c.type == 'tool_use').forEach((c: any) => {
+						merged.push({role: 'tool', id: c.id, name: c.name, args: c.input});
 					});
-				} else if(history[orgI].role == 'user') {
-					history[orgI].content.filter((c: any) => c.type =='tool_result').forEach((c: any) => {
-						const h = history.find((h: any) => h.id == c.tool_use_id);
-						h[c.is_error ? 'error' : 'content'] = c.content;
+				} else if(msg.role == 'user') {
+					msg.content.filter((c: any) => c.type == 'tool_result').forEach((c: any) => {
+						const h = merged.find((h: any) => h.id == c.tool_use_id);
+						if(h) h[c.is_error ? 'error' : 'content'] = c.content;
 					});
 				}
-				history[orgI].content = history[orgI].content.filter((c: any) => c.type == 'text').map((c: any) => c.text).join('\n\n');
+				msg.content = msg.content.filter((c: any) => c.type == 'text').map((c: any) => c.text).join('\n\n');
+			}
+			if(msg.content) {
+				const last = merged.at(-1);
+				if(last && last.role == 'assistant' && msg.role == 'assistant') last.content += '\n\n' + msg.content;
+				else merged.push({role: msg.role, content: msg.content});
 			}
 		}
-		return history.filter(h => !!h.content);
+		return merged;
 	}
 
 	private fromStandard(history: LLMMessage[]): any[] {
@@ -71,13 +76,15 @@ export class Anthropic extends LLMProvider {
 				stream: !!options.stream,
 			};
 
-			// Run tool changes
 			let resp: any;
+			let isFirstMessage = true;
 			do {
 				resp = await this.client.messages.create(requestParams);
 
-				// Streaming mode
 				if(options.stream) {
+					if(!isFirstMessage) options.stream({text: '\n\n'});
+					isFirstMessage = false;
+
 					resp.content = [];
 					for await (const chunk of resp) {
 						if(controller.signal.aborted) break;
@@ -104,7 +111,6 @@ export class Anthropic extends LLMProvider {
 					}
 				}
 
-				// Run tools
 				const toolCalls = resp.content.filter((c: any) => c.type === 'tool_use');
 				if(toolCalls.length && !controller.signal.aborted) {
 					history.push({role: 'assistant', content: resp.content});
@@ -122,12 +128,14 @@ export class Anthropic extends LLMProvider {
 					requestParams.messages = history;
 				}
 			} while (!controller.signal.aborted && resp.content.some((c: any) => c.type === 'tool_use'));
+
 			if(options.stream) options.stream({done: true});
 			res(this.toStandard([...history, {
 				role: 'assistant',
 				content: resp.content.filter((c: any) => c.type == 'text').map((c: any) => c.text).join('\n\n')
 			}]));
 		});
+
 		return Object.assign(response, {abort: () => controller.abort()});
 	}
 }