Use one-off workers to process requests without blocking

src/llm.ts

@@ -75,22 +75,10 @@ export type LLMRequest = {
 }
 
 class LLM {
-	private embedWorker: Worker | null = null;
-	private embedQueue = new Map<number, { resolve: (value: number[]) => void; reject: (error: any) => void }>();
-	private embedId = 0;
 	private models: {[model: string]: LLMProvider} = {};
 	private defaultModel!: string;
 
 	constructor(public readonly ai: Ai) {
-		this.embedWorker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'embedder.js'));
-		this.embedWorker.on('message', ({ id, embedding }) => {
-			const pending = this.embedQueue.get(id);
-			if (pending) {
-				pending.resolve(embedding);
-				this.embedQueue.delete(id);
-			}
-		});
-
 		if(!ai.options.llm?.models) return;
 		Object.entries(ai.options.llm.models).forEach(([model, config]) => {
 			if(!this.defaultModel) this.defaultModel = model;
@@ -269,14 +257,21 @@ class LLM {
 	embedding(target: object | string, maxTokens = 500, overlapTokens = 50) {
 		const embed = (text: string): Promise<number[]> => {
 			return new Promise((resolve, reject) => {
-				const id = this.embedId++;
-				this.embedQueue.set(id, { resolve, reject });
-				this.embedWorker?.postMessage({
-					id,
-					text,
-					model: this.ai.options?.embedder || 'bge-small-en-v1.5',
-					path: this.ai.options.path
+				const worker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'embedder.js'));
+				const handleMessage = ({ embedding }: any) => {
+					worker.terminate();
+					resolve(embedding);
+				};
+				const handleError = (err: Error) => {
+					worker.terminate();
+					reject(err);
+				};
+				worker.on('message', handleMessage);
+				worker.on('error', handleError);
+				worker.on('exit', (code) => {
+					if(code !== 0) reject(new Error(`Worker exited with code ${code}`));
 				});
+				worker.postMessage({text, model: this.ai.options?.embedder || 'bge-small-en-v1.5', path: this.ai.options.path});
 			});
 		};
 		const chunks = this.chunk(target, maxTokens, overlapTokens);