TTS

src/audio.ts

@@ -1,50 +1,80 @@
 import {spawn} from 'node:child_process';
+import * as os from 'node:os';
+import {platform, arch} from 'node:os';
 import fs from 'node:fs/promises';
 import Path from 'node:path';
-import {Ai} from './ai.ts';
+import {AbortablePromise, Ai} from './ai.ts';
 
 export class Audio {
 	private downloads: {[key: string]: Promise<string>} = {};
 	private whisperModel!: string;
+	private piperBinary?: string;
 
 	constructor(private ai: Ai) {
 		if(ai.options.whisper?.binary) {
 			this.whisperModel = ai.options.whisper?.model.endsWith('.bin') ? ai.options.whisper?.model : ai.options.whisper?.model + '.bin';
 			this.downloadAsrModel();
 		}
+		if(ai.options.piper?.model) {
+			if(!ai.options.piper.model.startsWith('http') || !ai.options.piper.model.endsWith('.onnx'))
+				throw new Error('Piper model should be a URL to an onnx file to download');
+			if(platform() != 'linux' || (arch() != 'x64' && arch() != 'arm64'))
+				throw new Error('Piper TTS only supported on Linux x64/arm64');
+			this.piperBinary = Path.join(import.meta.dirname, '../bin/piper');
+			this.downloadTtsModel();
+		}
 	}
 
-	/**
-	 * Convert audio to text using Auditory Speech Recognition
-	 * @param {string} path Path to audio
-	 * @param model Whisper model
-	 * @returns {Promise<any>} Extracted text
-	 */
-	asr(path: string, model: string = this.whisperModel): {abort: () => void, response: Promise<string | null>} {
+	asr(path: string, model: string = this.whisperModel): AbortablePromise<string | null> {
 		if(!this.ai.options.whisper?.binary) throw new Error('Whisper not configured');
 		let abort: any = () => {};
-		const response = new Promise<string | null>((resolve, reject) => {
-			this.downloadAsrModel(model).then(m => {
-				let output = '';
-				const proc = spawn(<string>this.ai.options.whisper?.binary, ['-nt', '-np', '-m', m, '-f', path], {stdio: ['ignore', 'pipe', 'ignore']});
-				abort = () => proc.kill('SIGTERM');
-				proc.on('error', (err: Error) => reject(err));
-				proc.stdout.on('data', (data: Buffer) => output += data.toString());
-				proc.on('close', (code: number) => {
-					if(code === 0) resolve(output.trim() || null);
-					else reject(new Error(`Exit code ${code}`));
-				});
+		const p = new Promise<string | null>(async (resolve, reject) => {
+			const m = await this.downloadAsrModel(model);
+			let output = '';
+			const proc = spawn(<string>this.ai.options.whisper?.binary, ['-nt', '-np', '-m', m, '-f', path], {stdio: ['ignore', 'pipe', 'ignore']});
+			abort = () => proc.kill('SIGTERM');
+			proc.on('error', (err: Error) => reject(err));
+			proc.stdout.on('data', (data: Buffer) => output += data.toString());
+			proc.on('close', (code: number) => {
+				if(code === 0) resolve(output.trim() || null);
+				else reject(new Error(`Exit code ${code}`));
 			});
 		});
-		return {response, abort};
+		return Object.assign(p, {abort});
+	}
+
+	tts(text: string, outputPath?: string, model: string = <string>this.ai.options.piper?.model): AbortablePromise<Buffer | string> {
+		if(!this.piperBinary) throw new Error('Piper not configured');
+		if(!model) throw new Error('Invalid Piper model');
+		let abort: any = () => {};
+		const p = new Promise<Buffer | string>(async (resolve, reject) => {
+			const modelPath = await this.downloadTtsModel(model);
+			const tmpFile = outputPath || Path.join(os.tmpdir(), `piper_${Date.now()}.wav`);
+			const proc = spawn(<string>this.piperBinary, ['--model', modelPath, '--output_file', tmpFile], {
+				stdio: ['pipe', 'ignore', 'ignore'],
+				env: {...process.env, LD_LIBRARY_PATH: Path.dirname(<string>this.piperBinary)}
+			});
+			abort = () => proc.kill('SIGTERM');
+			proc.stdin.write(text);
+			proc.stdin.end();
+			proc.on('error', (err: Error) => reject(err));
+			proc.on('close', async (code: number) => {
+				if(code === 0) {
+					if(outputPath) {
+						resolve(outputPath);
+					} else {
+						const buffer = await fs.readFile(tmpFile);
+						await fs.unlink(tmpFile).catch(() => {});
+						resolve(buffer);
+					}
+				} else {
+					reject(new Error(`Exit code ${code}`));
+				}
+			});
+		});
+		return Object.assign(p, {abort});
 	}
 
-	/**
-	 * Downloads the specified Whisper model if it is not already present locally.
-	 *
-	 * @param {string} model Whisper model that will be downloaded
-	 * @return {Promise<string>} Absolute path to model file, resolves once downloaded
-	 */
 	async downloadAsrModel(model: string = this.whisperModel): Promise<string> {
 		if(!this.ai.options.whisper?.binary) throw new Error('Whisper not configured');
 		if(!model.endsWith('.bin')) model += '.bin';
@@ -60,4 +90,24 @@ export class Audio {
 			});
 		return this.downloads[model];
 	}
+
+	async downloadTtsModel(model: string = <string>this.ai.options.piper?.model): Promise<string> {
+		if(!model) throw new Error('Invalid Piper model');
+		const m = <string>model.split('/').pop();
+		const p = Path.join(<string>this.ai.options.path, m);
+		const [onnxExists, jsonExists] = await Promise.all([
+			fs.stat(p).then(() => true).catch(() => false),
+			fs.stat(p + '.json').then(() => true).catch(() => false)
+		]);
+		if(onnxExists && jsonExists) return p;
+		if(!!this.downloads[m]) return this.downloads[m];
+		this.downloads[m] = Promise.all([
+			onnxExists ? Promise.resolve() : fetch(model).then(r => r.arrayBuffer()).then(b => fs.writeFile(p, Buffer.from(b))),
+			jsonExists ? Promise.resolve() : fetch(model + '.json').then(r => r.arrayBuffer()).then(b => fs.writeFile(p + '.json', Buffer.from(b)))
+		]).then(() => {
+			delete this.downloads[m];
+			return p;
+		});
+		return this.downloads[m];
+	}
 }