Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 473424ae23 | |||
| 9b831f7d95 |
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@ztimson/ai-utils",
|
"name": "@ztimson/ai-utils",
|
||||||
"version": "0.7.4",
|
"version": "0.7.6",
|
||||||
"description": "AI Utility library",
|
"description": "AI Utility library",
|
||||||
"author": "Zak Timson",
|
"author": "Zak Timson",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
|||||||
17
src/asr.ts
17
src/asr.ts
@@ -7,8 +7,6 @@ import { join } from 'node:path';
|
|||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import wavefile from 'wavefile';
|
import wavefile from 'wavefile';
|
||||||
|
|
||||||
let whisperPipeline: any;
|
|
||||||
|
|
||||||
export async function canDiarization(): Promise<string | null> {
|
export async function canDiarization(): Promise<string | null> {
|
||||||
const checkPython = (cmd: string) => {
|
const checkPython = (cmd: string) => {
|
||||||
return new Promise<boolean>((resolve) => {
|
return new Promise<boolean>((resolve) => {
|
||||||
@@ -110,30 +108,27 @@ function prepareAudioBuffer(file: string): [string, Float32Array] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
parentPort?.on('message', async ({ file, speaker, model, modelDir, token }) => {
|
parentPort?.on('message', async ({ file, speaker, model, modelDir, token }) => {
|
||||||
|
let tempFile = null;
|
||||||
try {
|
try {
|
||||||
if(!whisperPipeline) whisperPipeline = await pipeline('automatic-speech-recognition', `Xenova/${model}`, {cache_dir: modelDir, quantized: true});
|
const asr: any = await pipeline('automatic-speech-recognition', `Xenova/${model}`, {cache_dir: modelDir, quantized: true});
|
||||||
|
|
||||||
// Prepare audio file
|
|
||||||
const [f, buffer] = prepareAudioBuffer(file);
|
const [f, buffer] = prepareAudioBuffer(file);
|
||||||
|
tempFile = f !== file ? f : null;
|
||||||
// Fetch transcript and speakers
|
|
||||||
const hasDiarization = await canDiarization();
|
const hasDiarization = await canDiarization();
|
||||||
const [transcript, speakers] = await Promise.all([
|
const [transcript, speakers] = await Promise.all([
|
||||||
whisperPipeline(buffer, {return_timestamps: speaker ? 'word' : false}),
|
asr(buffer, {return_timestamps: speaker ? 'word' : false}),
|
||||||
(!speaker || !token || !hasDiarization) ? Promise.resolve(): runDiarization(hasDiarization, f, modelDir, token),
|
(!speaker || !token || !hasDiarization) ? Promise.resolve(): runDiarization(hasDiarization, f, modelDir, token),
|
||||||
]);
|
]);
|
||||||
if(file != f) rmSync(f, { recursive: true, force: true });
|
|
||||||
|
|
||||||
// Return any results / errors if no more processing required
|
|
||||||
const text = transcript.text?.trim() || null;
|
const text = transcript.text?.trim() || null;
|
||||||
if(!speaker) return parentPort?.postMessage({ text });
|
if(!speaker) return parentPort?.postMessage({ text });
|
||||||
if(!token) return parentPort?.postMessage({ text, error: 'HuggingFace token required' });
|
if(!token) return parentPort?.postMessage({ text, error: 'HuggingFace token required' });
|
||||||
if(!hasDiarization) return parentPort?.postMessage({ text, error: 'Speaker diarization unavailable' });
|
if(!hasDiarization) return parentPort?.postMessage({ text, error: 'Speaker diarization unavailable' });
|
||||||
|
|
||||||
// Combine transcript and speakers
|
|
||||||
const combined = combineSpeakerTranscript(transcript.chunks || [], speakers || []);
|
const combined = combineSpeakerTranscript(transcript.chunks || [], speakers || []);
|
||||||
parentPort?.postMessage({ text: combined });
|
parentPort?.postMessage({ text: combined });
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
parentPort?.postMessage({ error: err.stack || err.message });
|
parentPort?.postMessage({ error: err.stack || err.message });
|
||||||
|
} finally {
|
||||||
|
if(tempFile) rmSync(tempFile, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ export class Audio {
|
|||||||
let p = new Promise<string | null>((resolve, reject) => {
|
let p = new Promise<string | null>((resolve, reject) => {
|
||||||
const worker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'asr.js'));
|
const worker = new Worker(join(dirname(fileURLToPath(import.meta.url)), 'asr.js'));
|
||||||
const handleMessage = ({ text, warning, error }: any) => {
|
const handleMessage = ({ text, warning, error }: any) => {
|
||||||
worker.terminate();
|
setTimeout(() => worker.terminate(), 1000);
|
||||||
if(aborted) return;
|
if(aborted) return;
|
||||||
if(error) reject(new Error(error));
|
if(error) reject(new Error(error));
|
||||||
else {
|
else {
|
||||||
@@ -24,7 +24,7 @@ export class Audio {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
const handleError = (err: Error) => {
|
const handleError = (err: Error) => {
|
||||||
worker.terminate();
|
setTimeout(() => worker.terminate(), 1000);
|
||||||
if(!aborted) reject(err);
|
if(!aborted) reject(err);
|
||||||
};
|
};
|
||||||
worker.on('message', handleMessage);
|
worker.on('message', handleMessage);
|
||||||
@@ -42,8 +42,8 @@ export class Audio {
|
|||||||
if(!transcript) return transcript;
|
if(!transcript) return transcript;
|
||||||
let chunks = this.ai.language.chunk(transcript, 500, 0);
|
let chunks = this.ai.language.chunk(transcript, 500, 0);
|
||||||
if(chunks.length > 4) chunks = [...chunks.slice(0, 3), <string>chunks.at(-1)];
|
if(chunks.length > 4) chunks = [...chunks.slice(0, 3), <string>chunks.at(-1)];
|
||||||
const names = await this.ai.language.json(chunks.join('\n'), '{1: "Detected Name"}', {
|
const names = await this.ai.language.json(chunks.join('\n'), '{1: "Detected Name", 2: "Second Name"}', {
|
||||||
system: 'Use this following transcript to identify speakers. Only identify speakers you are sure about',
|
system: 'Use the following transcript to identify speakers. Only identify speakers you are positive about, dont mention speakers you are unsure about in your response',
|
||||||
temperature: 0.1,
|
temperature: 0.1,
|
||||||
});
|
});
|
||||||
Object.entries(names).forEach(([speaker, name]) => {
|
Object.entries(names).forEach(([speaker, name]) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user