|
|
|
@@ -1,7 +1,8 @@
|
|
|
|
import {execSync, spawn} from 'node:child_process';
|
|
|
|
import {execSync, spawn} from 'node:child_process';
|
|
|
|
import {mkdtempSync, rmSync} from 'node:fs';
|
|
|
|
import {mkdtempSync} from 'node:fs';
|
|
|
|
import fs from 'node:fs/promises';
|
|
|
|
import fs from 'node:fs/promises';
|
|
|
|
import {tmpdir} from 'node:os';
|
|
|
|
import {tmpdir} from 'node:os';
|
|
|
|
|
|
|
|
import * as path from 'node:path';
|
|
|
|
import Path, {join} from 'node:path';
|
|
|
|
import Path, {join} from 'node:path';
|
|
|
|
import {AbortablePromise, Ai} from './ai.ts';
|
|
|
|
import {AbortablePromise, Ai} from './ai.ts';
|
|
|
|
|
|
|
|
|
|
|
|
@@ -12,7 +13,7 @@ export class Audio {
|
|
|
|
|
|
|
|
|
|
|
|
constructor(private ai: Ai) {
|
|
|
|
constructor(private ai: Ai) {
|
|
|
|
if(ai.options.whisper) {
|
|
|
|
if(ai.options.whisper) {
|
|
|
|
this.whisperModel = ai.options.asr?.endsWith('.bin') ? ai.options.asr : ai.options.asr + '.bin';
|
|
|
|
this.whisperModel = ai.options.asr || 'ggml-base.en.bin';
|
|
|
|
this.downloadAsrModel();
|
|
|
|
this.downloadAsrModel();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@@ -38,23 +39,36 @@ print(json.dumps(segments))
|
|
|
|
let proc: any;
|
|
|
|
let proc: any;
|
|
|
|
const p = new Promise<any>((resolve, reject) => {
|
|
|
|
const p = new Promise<any>((resolve, reject) => {
|
|
|
|
this.downloadAsrModel(opts.model).then(m => {
|
|
|
|
this.downloadAsrModel(opts.model).then(m => {
|
|
|
|
let output = '';
|
|
|
|
if(opts.diarization) {
|
|
|
|
const args = [opts.diarization ? '-owts' : '-nt', '-m', m, '-f', file];
|
|
|
|
let output = path.join(path.dirname(file), 'transcript');
|
|
|
|
proc = spawn(<string>this.ai.options.whisper, args, {stdio: ['ignore', 'pipe', 'ignore']});
|
|
|
|
proc = spawn(<string>this.ai.options.whisper,
|
|
|
|
proc.on('error', (err: Error) => reject(err));
|
|
|
|
['-m', m, '-f', file, '-np', '-ml', '1', '-oj', '-of', output],
|
|
|
|
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
|
|
|
{stdio: ['ignore', 'ignore', 'pipe']}
|
|
|
|
proc.on('close', (code: number) => {
|
|
|
|
);
|
|
|
|
if(code === 0) {
|
|
|
|
proc.on('error', (err: Error) => reject(err));
|
|
|
|
if(opts.diarization) {
|
|
|
|
proc.on('close', async (code: number) => {
|
|
|
|
|
|
|
|
if(code === 0) {
|
|
|
|
|
|
|
|
output = await fs.readFile(output + '.json', 'utf-8');
|
|
|
|
|
|
|
|
fs.rm(output + '.json').catch(() => { });
|
|
|
|
try { resolve(JSON.parse(output)); }
|
|
|
|
try { resolve(JSON.parse(output)); }
|
|
|
|
catch(e) { reject(new Error('Failed to parse whisper JSON')); }
|
|
|
|
catch(e) { reject(new Error('Failed to parse whisper JSON')); }
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
resolve(output.trim() || null);
|
|
|
|
reject(new Error(`Exit code ${code}`));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
});
|
|
|
|
reject(new Error(`Exit code ${code}`));
|
|
|
|
} else {
|
|
|
|
}
|
|
|
|
let output = '';
|
|
|
|
});
|
|
|
|
proc = spawn(<string>this.ai.options.whisper, ['-m', m, '-f', file, '-np', '-nt']);
|
|
|
|
|
|
|
|
proc.on('error', (err: Error) => reject(err));
|
|
|
|
|
|
|
|
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
|
|
|
|
|
|
|
proc.on('close', async (code: number) => {
|
|
|
|
|
|
|
|
if(code === 0) {
|
|
|
|
|
|
|
|
resolve(output.trim() || null);
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
reject(new Error(`Exit code ${code}`));
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
return <any>Object.assign(p, {abort: () => proc?.kill('SIGTERM')});
|
|
|
|
return <any>Object.assign(p, {abort: () => proc?.kill('SIGTERM')});
|
|
|
|
@@ -64,7 +78,7 @@ print(json.dumps(segments))
|
|
|
|
let aborted = false, abort = () => { aborted = true; };
|
|
|
|
let aborted = false, abort = () => { aborted = true; };
|
|
|
|
const checkPython = (cmd: string) => {
|
|
|
|
const checkPython = (cmd: string) => {
|
|
|
|
return new Promise<boolean>((resolve) => {
|
|
|
|
return new Promise<boolean>((resolve) => {
|
|
|
|
const proc = spawn(cmd, ['-c', 'import pyannote.audio']);
|
|
|
|
const proc = spawn(cmd, ['-W', 'ignore', '-c', 'import pyannote.audio']);
|
|
|
|
proc.on('close', (code: number) => resolve(code === 0));
|
|
|
|
proc.on('close', (code: number) => resolve(code === 0));
|
|
|
|
proc.on('error', () => resolve(false));
|
|
|
|
proc.on('error', () => resolve(false));
|
|
|
|
});
|
|
|
|
});
|
|
|
|
@@ -76,13 +90,10 @@ print(json.dumps(segments))
|
|
|
|
if(aborted) return;
|
|
|
|
if(aborted) return;
|
|
|
|
if(!p && !p3) throw new Error('Pyannote is not installed: pip install pyannote.audio');
|
|
|
|
if(!p && !p3) throw new Error('Pyannote is not installed: pip install pyannote.audio');
|
|
|
|
const binary = p3 ? 'python3' : 'python';
|
|
|
|
const binary = p3 ? 'python3' : 'python';
|
|
|
|
let tmp: string | null = null;
|
|
|
|
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
tmp = join(mkdtempSync(join(tmpdir(), 'audio-')), 'converted.wav');
|
|
|
|
|
|
|
|
execSync(`ffmpeg -i "${file}" -ar 16000 -ac 1 -f wav "${tmp}"`, { stdio: 'ignore' });
|
|
|
|
|
|
|
|
if(aborted) return;
|
|
|
|
if(aborted) return;
|
|
|
|
let output = '';
|
|
|
|
let output = '';
|
|
|
|
const proc = spawn(binary, ['-c', this.pyannote, tmp]);
|
|
|
|
const proc = spawn(binary, ['-W', 'ignore', '-c', this.pyannote, file]);
|
|
|
|
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
|
|
|
proc.stdout.on('data', (data: Buffer) => output += data.toString());
|
|
|
|
proc.stderr.on('data', (data: Buffer) => console.error(data.toString()));
|
|
|
|
proc.stderr.on('data', (data: Buffer) => console.error(data.toString()));
|
|
|
|
proc.on('close', (code: number) => {
|
|
|
|
proc.on('close', (code: number) => {
|
|
|
|
@@ -95,62 +106,108 @@ print(json.dumps(segments))
|
|
|
|
});
|
|
|
|
});
|
|
|
|
proc.on('error', reject);
|
|
|
|
proc.on('error', reject);
|
|
|
|
abort = () => proc.kill('SIGTERM');
|
|
|
|
abort = () => proc.kill('SIGTERM');
|
|
|
|
}).finally(() => { if(tmp) rmSync(Path.dirname(tmp), { recursive: true, force: true }); });
|
|
|
|
});
|
|
|
|
}));
|
|
|
|
}));
|
|
|
|
return <any>Object.assign(p, {abort});
|
|
|
|
return <any>Object.assign(p, {abort});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private combineSpeakerTranscript(transcript: any, speakers: any[]): string {
|
|
|
|
private async combineSpeakerTranscript(punctuatedText: string, timestampData: any, speakers: any[]): Promise<string> {
|
|
|
|
const speakerMap = new Map();
|
|
|
|
const speakerMap = new Map();
|
|
|
|
let speakerCount = 0;
|
|
|
|
let speakerCount = 0;
|
|
|
|
speakers.forEach((seg: any) => {
|
|
|
|
speakers.forEach((seg: any) => {
|
|
|
|
if(!speakerMap.has(seg.speaker)) speakerMap.set(seg.speaker, ++speakerCount);
|
|
|
|
if(!speakerMap.has(seg.speaker)) speakerMap.set(seg.speaker, ++speakerCount);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const sentences = punctuatedText.match(/[^.!?]+[.!?]+/g) || [punctuatedText];
|
|
|
|
const lines: string[] = [];
|
|
|
|
const lines: string[] = [];
|
|
|
|
let currentSpeaker = -1;
|
|
|
|
|
|
|
|
let currentText = '';
|
|
|
|
sentences.forEach(sentence => {
|
|
|
|
transcript.transcription.forEach((word: any) => {
|
|
|
|
sentence = sentence.trim();
|
|
|
|
const time = word.offsets.from / 1000; // Convert ms to seconds
|
|
|
|
if(!sentence) return;
|
|
|
|
const speaker = speakers.find((s: any) => time >= s.start && time <= s.end);
|
|
|
|
|
|
|
|
const speakerNum = speaker ? speakerMap.get(speaker.speaker) : 1;
|
|
|
|
const words = sentence.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
|
|
|
|
if (speakerNum !== currentSpeaker) {
|
|
|
|
let startTime = Infinity, endTime = 0;
|
|
|
|
if(currentText) lines.push(`[Speaker ${currentSpeaker}]: ${currentText.trim()}`);
|
|
|
|
const wordTimings: {start: number, end: number}[] = [];
|
|
|
|
currentSpeaker = speakerNum;
|
|
|
|
|
|
|
|
currentText = word.text;
|
|
|
|
timestampData.transcription.forEach((word: any) => {
|
|
|
|
} else {
|
|
|
|
const wordText = word.text.trim().toLowerCase();
|
|
|
|
currentText += ' ' + word.text;
|
|
|
|
if(words.some(w => wordText.includes(w))) {
|
|
|
|
}
|
|
|
|
const start = word.offsets.from / 1000;
|
|
|
|
|
|
|
|
const end = word.offsets.to / 1000;
|
|
|
|
|
|
|
|
wordTimings.push({start, end});
|
|
|
|
|
|
|
|
if(start < startTime) startTime = start;
|
|
|
|
|
|
|
|
if(end > endTime) endTime = end;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(startTime === Infinity) return;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Weight by word-level overlap instead of sentence span
|
|
|
|
|
|
|
|
const speakerScores = new Map<number, number>();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
wordTimings.forEach(wt => {
|
|
|
|
|
|
|
|
speakers.forEach((seg: any) => {
|
|
|
|
|
|
|
|
const overlap = Math.max(0, Math.min(wt.end, seg.end) - Math.max(wt.start, seg.start));
|
|
|
|
|
|
|
|
const duration = wt.end - wt.start;
|
|
|
|
|
|
|
|
if(duration > 0) {
|
|
|
|
|
|
|
|
const score = overlap / duration; // % of word covered
|
|
|
|
|
|
|
|
const spkNum = speakerMap.get(seg.speaker);
|
|
|
|
|
|
|
|
speakerScores.set(spkNum, (speakerScores.get(spkNum) || 0) + score);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let bestSpeaker = 1;
|
|
|
|
|
|
|
|
let maxScore = 0;
|
|
|
|
|
|
|
|
speakerScores.forEach((score, speaker) => {
|
|
|
|
|
|
|
|
if(score > maxScore) {
|
|
|
|
|
|
|
|
maxScore = score;
|
|
|
|
|
|
|
|
bestSpeaker = speaker;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lines.push(`[Speaker ${bestSpeaker}]: ${sentence}`);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
if(currentText) lines.push(`[Speaker ${currentSpeaker}]: ${currentText.trim()}`);
|
|
|
|
|
|
|
|
return lines.join('\n');
|
|
|
|
return lines.join('\n').trim();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
asr(file: string, options: { model?: string; diarization?: boolean | 'id' } = {}): AbortablePromise<string | null> {
|
|
|
|
asr(file: string, options: { model?: string; diarization?: boolean | 'id' } = {}): AbortablePromise<string | null> {
|
|
|
|
if(!this.ai.options.whisper) throw new Error('Whisper not configured');
|
|
|
|
if(!this.ai.options.whisper) throw new Error('Whisper not configured');
|
|
|
|
|
|
|
|
|
|
|
|
const transcript = this.runAsr(file, {model: options.model, diarization: !!options.diarization});
|
|
|
|
const tmp = join(mkdtempSync(join(tmpdir(), 'audio-')), 'converted.wav');
|
|
|
|
const diarization: any = options.diarization ? this.runDiarization(file) : Promise.resolve(null);
|
|
|
|
execSync(`ffmpeg -i "${file}" -ar 16000 -ac 1 -f wav "${tmp}"`, { stdio: 'ignore' });
|
|
|
|
const abort = () => {
|
|
|
|
const clean = () => fs.rm(Path.dirname(tmp), {recursive: true, force: true}).catch(() => {});
|
|
|
|
|
|
|
|
const transcript = this.runAsr(tmp, {model: options.model, diarization: false});
|
|
|
|
|
|
|
|
const timestamps: any = !options.diarization ? Promise.resolve(null) : this.runAsr(tmp, {model: options.model, diarization: true});
|
|
|
|
|
|
|
|
const diarization: any = !options.diarization ? Promise.resolve(null) : this.runDiarization(tmp);
|
|
|
|
|
|
|
|
let aborted = false, abort = () => {
|
|
|
|
|
|
|
|
aborted = true;
|
|
|
|
transcript.abort();
|
|
|
|
transcript.abort();
|
|
|
|
|
|
|
|
timestamps?.abort?.();
|
|
|
|
diarization?.abort?.();
|
|
|
|
diarization?.abort?.();
|
|
|
|
|
|
|
|
clean();
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
const response = Promise.all([transcript, diarization]).then(async ([t, d]) => {
|
|
|
|
const response = Promise.allSettled([transcript, timestamps, diarization]).then(async ([t, ts, d]) => {
|
|
|
|
if(!options.diarization) return t;
|
|
|
|
if(t.status == 'rejected') throw new Error('Whisper.cpp punctuated:\n' + t.reason);
|
|
|
|
t = this.combineSpeakerTranscript(t, d);
|
|
|
|
if(ts.status == 'rejected') throw new Error('Whisper.cpp timestamps:\n' + ts.reason);
|
|
|
|
if(options.diarization === 'id') {
|
|
|
|
if(d.status == 'rejected') throw new Error('Pyannote:\n' + d.reason);
|
|
|
|
|
|
|
|
if(aborted || !options.diarization) return t.value;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let transcript = await this.combineSpeakerTranscript(t.value, ts.value, d.value);
|
|
|
|
|
|
|
|
if(!aborted && options.diarization === 'id') {
|
|
|
|
if(!this.ai.language.defaultModel) throw new Error('Configure an LLM for advanced ASR speaker detection');
|
|
|
|
if(!this.ai.language.defaultModel) throw new Error('Configure an LLM for advanced ASR speaker detection');
|
|
|
|
let chunks = this.ai.language.chunk(t, 500, 0);
|
|
|
|
let chunks = this.ai.language.chunk(transcript, 500, 0);
|
|
|
|
if(chunks.length > 4) chunks = [...chunks.slice(0, 3), <string>chunks.at(-1)];
|
|
|
|
if(chunks.length > 4) chunks = [...chunks.slice(0, 3), <string>chunks.at(-1)];
|
|
|
|
const names = await this.ai.language.json(chunks.join('\n'), '{1: "Detected Name", 2: "Second Name"}', {
|
|
|
|
const names = await this.ai.language.json(chunks.join('\n'), '{1: "Detected Name", 2: "Second Name"}', {
|
|
|
|
system: 'Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response',
|
|
|
|
system: 'Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response',
|
|
|
|
temperature: 0.1,
|
|
|
|
temperature: 0.1,
|
|
|
|
});
|
|
|
|
});
|
|
|
|
Object.entries(names).forEach(([speaker, name]) => t = t.replaceAll(`[Speaker ${speaker}]`, `[${name}]`));
|
|
|
|
Object.entries(names).forEach(([speaker, name]) => transcript = transcript.replaceAll(`[Speaker ${speaker}]`, `[${name}]`));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return t;
|
|
|
|
return transcript;
|
|
|
|
});
|
|
|
|
}).finally(() => clean());
|
|
|
|
return <any>Object.assign(response, {abort});
|
|
|
|
return <any>Object.assign(response, {abort});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
