feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)
Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
This commit is contained in:
Bohdan Triapitsyn
committed by
GitHub
GitHub
parent
3f5151d424
commit
de1b85ac56
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Catalog of local sherpa-onnx STT models available for dictation.
|
||||
* Models are downloaded on demand from the k2-fsa GitHub releases and
|
||||
* extracted under the OpenChamber speech-models directory.
|
||||
*
|
||||
* `type` selects the recognizer construction path in the worker:
|
||||
* - 'nemo_transducer': encoder/decoder/joiner transducer (Parakeet)
|
||||
* - 'whisper': encoder/decoder Whisper export
|
||||
* `files` maps logical roles to file names inside the extracted directory.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
|
||||
export const LOCAL_STT_MODEL_CATALOG = {
|
||||
'parakeet-tdt-0.6b-v2-int8': {
|
||||
type: 'nemo_transducer',
|
||||
archiveUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2',
|
||||
extractedDir: 'sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8',
|
||||
files: {
|
||||
encoder: 'encoder.int8.onnx',
|
||||
decoder: 'decoder.int8.onnx',
|
||||
joiner: 'joiner.int8.onnx',
|
||||
tokens: 'tokens.txt',
|
||||
},
|
||||
description: 'NVIDIA Parakeet TDT v2 (English)',
|
||||
},
|
||||
'parakeet-tdt-0.6b-v3-int8': {
|
||||
type: 'nemo_transducer',
|
||||
archiveUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2',
|
||||
extractedDir: 'sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8',
|
||||
files: {
|
||||
encoder: 'encoder.int8.onnx',
|
||||
decoder: 'decoder.int8.onnx',
|
||||
joiner: 'joiner.int8.onnx',
|
||||
tokens: 'tokens.txt',
|
||||
},
|
||||
description: 'NVIDIA Parakeet TDT v3 (25 European languages, auto-detected)',
|
||||
},
|
||||
'whisper-base-int8': {
|
||||
type: 'whisper',
|
||||
archiveUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-base.tar.bz2',
|
||||
extractedDir: 'sherpa-onnx-whisper-base',
|
||||
files: {
|
||||
encoder: 'base-encoder.int8.onnx',
|
||||
decoder: 'base-decoder.int8.onnx',
|
||||
tokens: 'base-tokens.txt',
|
||||
},
|
||||
description: 'OpenAI Whisper base (multilingual, smaller and lighter)',
|
||||
},
|
||||
'whisper-tiny-int8': {
|
||||
type: 'whisper',
|
||||
archiveUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2',
|
||||
extractedDir: 'sherpa-onnx-whisper-tiny',
|
||||
files: {
|
||||
encoder: 'tiny-encoder.int8.onnx',
|
||||
decoder: 'tiny-decoder.int8.onnx',
|
||||
tokens: 'tiny-tokens.txt',
|
||||
},
|
||||
description: 'OpenAI Whisper tiny (multilingual, fastest and lightest)',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Local text-to-speech models (sherpa-onnx OfflineTts). Downloaded and
|
||||
* managed through the same pipeline as the STT models.
|
||||
*/
|
||||
export const LOCAL_TTS_MODEL_CATALOG = {
|
||||
'kokoro-en-v0_19': {
|
||||
type: 'kokoro',
|
||||
archiveUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2',
|
||||
extractedDir: 'kokoro-en-v0_19',
|
||||
files: {
|
||||
model: 'model.onnx',
|
||||
voices: 'voices.bin',
|
||||
tokens: 'tokens.txt',
|
||||
espeakData: 'espeak-ng-data',
|
||||
},
|
||||
description: 'Kokoro TTS (English, natural voices)',
|
||||
},
|
||||
};
|
||||
|
||||
export const DEFAULT_LOCAL_STT_MODEL = 'parakeet-tdt-0.6b-v2-int8';
|
||||
export const DEFAULT_LOCAL_TTS_MODEL = 'kokoro-en-v0_19';
|
||||
|
||||
export const LOCAL_STT_MODEL_IDS = Object.keys(LOCAL_STT_MODEL_CATALOG);
|
||||
export const LOCAL_TTS_MODEL_IDS = Object.keys(LOCAL_TTS_MODEL_CATALOG);
|
||||
|
||||
/**
|
||||
* @param {string} modelId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLocalSttModelId(modelId) {
|
||||
return typeof modelId === 'string' && Object.hasOwn(LOCAL_STT_MODEL_CATALOG, modelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} modelId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLocalTtsModelId(modelId) {
|
||||
return typeof modelId === 'string' && Object.hasOwn(LOCAL_TTS_MODEL_CATALOG, modelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Any managed local model (STT or TTS).
|
||||
* @param {string} modelId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLocalModelId(modelId) {
|
||||
return isLocalSttModelId(modelId) || isLocalTtsModelId(modelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Spec lookup across both catalogs (STT and TTS).
|
||||
* @param {string} modelId
|
||||
*/
|
||||
export function getLocalSttModelSpec(modelId) {
|
||||
const spec = LOCAL_STT_MODEL_CATALOG[modelId] ?? LOCAL_TTS_MODEL_CATALOG[modelId];
|
||||
if (!spec) {
|
||||
throw new Error(`Unknown local speech model id: ${modelId}`);
|
||||
}
|
||||
return {
|
||||
id: modelId,
|
||||
...spec,
|
||||
requiredFiles: Object.values(spec.files),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} modelsDir
|
||||
* @param {string} modelId
|
||||
* @returns {string}
|
||||
*/
|
||||
export function getLocalSttModelDir(modelsDir, modelId) {
|
||||
return path.join(modelsDir, getLocalSttModelSpec(modelId).extractedDir);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Downloads and extracts local sherpa-onnx STT model archives.
|
||||
* Archives (.tar.bz2) come from the k2-fsa GitHub releases and are extracted
|
||||
* with the system `tar` into the speech-models directory.
|
||||
*/
|
||||
|
||||
import { createWriteStream } from 'fs';
|
||||
import { mkdir, rename, rm, stat } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { spawn } from 'child_process';
|
||||
|
||||
import { getLocalSttModelSpec } from './model-catalog.js';
|
||||
|
||||
async function hasRequiredFiles(modelDir, requiredFiles) {
|
||||
const results = await Promise.all(
|
||||
requiredFiles.map(async (rel) => {
|
||||
try {
|
||||
const s = await stat(path.join(modelDir, rel));
|
||||
if (s.isDirectory()) {
|
||||
return true;
|
||||
}
|
||||
return s.isFile() && s.size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}),
|
||||
);
|
||||
return results.every(Boolean);
|
||||
}
|
||||
|
||||
async function downloadToFile(url, outputPath, onProgress) {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
if (!res.body) {
|
||||
throw new Error(`Failed to download ${url}: missing response body`);
|
||||
}
|
||||
|
||||
const totalBytes = Number.parseInt(res.headers.get('content-length') || '', 10) || null;
|
||||
let downloadedBytes = 0;
|
||||
|
||||
const tmpPath = `${outputPath}.tmp-${Date.now()}`;
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
|
||||
const nodeStream = Readable.fromWeb(res.body);
|
||||
if (typeof onProgress === 'function') {
|
||||
nodeStream.on('data', (chunk) => {
|
||||
downloadedBytes += chunk.length;
|
||||
onProgress(downloadedBytes, totalBytes);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
await pipeline(nodeStream, createWriteStream(tmpPath));
|
||||
await rename(tmpPath, outputPath);
|
||||
} catch (error) {
|
||||
await rm(tmpPath, { force: true }).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function extractTarArchive(archivePath, destDir) {
|
||||
await mkdir(destDir, { recursive: true });
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const child = spawn('tar', ['xf', archivePath, '-C', destDir], {
|
||||
stdio: 'ignore',
|
||||
windowsHide: true,
|
||||
});
|
||||
child.on('error', reject);
|
||||
child.on('exit', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`tar exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function isNonEmptyFile(filePath) {
|
||||
try {
|
||||
const s = await stat(filePath);
|
||||
return s.isFile() && s.size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a model is fully installed (all required files present).
|
||||
* @param {string} modelsDir
|
||||
* @param {string} modelId
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
export async function isLocalSttModelInstalled(modelsDir, modelId) {
|
||||
const spec = getLocalSttModelSpec(modelId);
|
||||
return hasRequiredFiles(path.join(modelsDir, spec.extractedDir), spec.requiredFiles);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a model is downloaded and extracted. Resolves with the model dir.
|
||||
*
|
||||
* Extraction is staged: the archive unpacks into a temporary directory and is
|
||||
* verified before being renamed into place. An interrupted or failed tar must
|
||||
* never leave partial files at the final path — the installed check only
|
||||
* verifies file presence, so a truncated .onnx there would be treated as an
|
||||
* installed model forever ("Protobuf parsing failed" at load time).
|
||||
*
|
||||
* @param {{ modelsDir: string, modelId: string,
|
||||
* onProgress?: (downloadedBytes: number, totalBytes: number | null) => void }} options
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
export async function ensureLocalSttModel({ modelsDir, modelId, onProgress }) {
|
||||
const spec = getLocalSttModelSpec(modelId);
|
||||
const modelDir = path.join(modelsDir, spec.extractedDir);
|
||||
if (await hasRequiredFiles(modelDir, spec.requiredFiles)) {
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
// A directory that exists but fails the required-files check is a partial
|
||||
// extraction from an earlier interrupted attempt — remove it before retrying.
|
||||
await rm(modelDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
|
||||
const downloadsDir = path.join(modelsDir, '.downloads');
|
||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||
|
||||
if (!(await isNonEmptyFile(archivePath))) {
|
||||
await downloadToFile(spec.archiveUrl, archivePath, onProgress);
|
||||
}
|
||||
|
||||
const stagingDir = path.join(modelsDir, `.staging-${spec.extractedDir}-${Date.now()}`);
|
||||
try {
|
||||
await extractTarArchive(archivePath, stagingDir);
|
||||
|
||||
const stagedModelDir = path.join(stagingDir, spec.extractedDir);
|
||||
if (!(await hasRequiredFiles(stagedModelDir, spec.requiredFiles))) {
|
||||
// Bad archive (truncated download / corrupt cache): drop it so the next
|
||||
// attempt re-downloads instead of re-extracting the same broken bytes.
|
||||
await rm(archivePath, { force: true }).catch(() => undefined);
|
||||
throw new Error(
|
||||
`Extracted ${archiveFilename}, but required model files are missing or empty. The archive was discarded; retry to re-download.`,
|
||||
);
|
||||
}
|
||||
|
||||
await rename(stagedModelDir, modelDir);
|
||||
} catch (error) {
|
||||
await rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
// Any extraction failure means the cached archive can't be trusted
|
||||
// (corrupt bz2, truncated download). Discard it so retry re-downloads.
|
||||
await rm(archivePath, { force: true }).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
await rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
|
||||
await rm(archivePath, { force: true }).catch(() => undefined);
|
||||
|
||||
return modelDir;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* Loader for the sherpa-onnx-node native addon.
|
||||
*
|
||||
* sherpa-onnx-node ships its native addon and shared libraries in a
|
||||
* platform-specific package (e.g. sherpa-onnx-darwin-arm64). The shared
|
||||
* libraries must be findable via the platform's dynamic-loader search path,
|
||||
* so the loader prepends the platform package directory to LD_LIBRARY_PATH /
|
||||
* DYLD_LIBRARY_PATH / PATH before requiring the addon.
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import path from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
let cached = null;
|
||||
|
||||
function sherpaPlatformPackageName(platform = process.platform, arch = process.arch) {
|
||||
const normalizedPlatform = platform === 'win32' ? 'win' : platform;
|
||||
return `sherpa-onnx-${normalizedPlatform}-${arch}`;
|
||||
}
|
||||
|
||||
function sherpaLoaderEnvKey(platform = process.platform) {
|
||||
if (platform === 'linux') {
|
||||
return 'LD_LIBRARY_PATH';
|
||||
}
|
||||
if (platform === 'darwin') {
|
||||
return 'DYLD_LIBRARY_PATH';
|
||||
}
|
||||
if (platform === 'win32') {
|
||||
return 'PATH';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function prependEnvPath(existing, value) {
|
||||
const parts = String(existing ?? '').split(path.delimiter).filter(Boolean);
|
||||
if (parts.includes(value)) {
|
||||
return parts.join(path.delimiter);
|
||||
}
|
||||
return [value, ...parts].join(path.delimiter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Case-insensitive env key lookup: on Windows `{...process.env}` yields a
|
||||
* plain object where PATH may be stored as `Path`. Using a hardcoded 'PATH'
|
||||
* would create a duplicate key and break the child process PATH.
|
||||
*/
|
||||
function findEnvKey(env, key) {
|
||||
const lower = key.toLowerCase();
|
||||
for (const k of Object.keys(env)) {
|
||||
if (k.toLowerCase() === lower) {
|
||||
return k;
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
function resolveSherpaLibDir(platform = process.platform, arch = process.arch) {
|
||||
const packageName = sherpaPlatformPackageName(platform, arch);
|
||||
try {
|
||||
const pkgJson = require.resolve(`${packageName}/package.json`);
|
||||
// Electron packages node_modules inside app.asar, but native addons and
|
||||
// their shared libraries are extracted to app.asar.unpacked. The dynamic
|
||||
// loader (dlopen/DYLD/LD) cannot read from the asar archive, so point the
|
||||
// search path at the unpacked copy.
|
||||
const dir = path.dirname(pkgJson);
|
||||
const unpacked = dir.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`);
|
||||
return existsSync(unpacked) ? unpacked : dir;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepend the sherpa platform package dir to the loader search path env var.
|
||||
* Mutates the provided env object.
|
||||
* @param {NodeJS.ProcessEnv} env
|
||||
*/
|
||||
export function applySherpaLoaderEnv(env) {
|
||||
const key = sherpaLoaderEnvKey();
|
||||
const libDir = resolveSherpaLibDir();
|
||||
if (!key || !libDir) {
|
||||
return { key: null, libDir: null };
|
||||
}
|
||||
const actualKey = findEnvKey(env, key);
|
||||
env[actualKey] = prependEnvPath(env[actualKey], libDir);
|
||||
return { key, libDir };
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the sherpa-onnx-node module, trying the upstream entry first and then
|
||||
* the platform addon directly.
|
||||
*/
|
||||
export function loadSherpaOnnxNode() {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const attempts = [];
|
||||
|
||||
try {
|
||||
cached = require('sherpa-onnx-node');
|
||||
return cached;
|
||||
} catch (error) {
|
||||
attempts.push(`sherpa-onnx-node: ${error?.message || String(error)}`);
|
||||
}
|
||||
|
||||
const libDir = resolveSherpaLibDir();
|
||||
if (libDir) {
|
||||
applySherpaLoaderEnv(process.env);
|
||||
const addonPath = path.join(libDir, 'sherpa-onnx.node');
|
||||
if (existsSync(addonPath)) {
|
||||
try {
|
||||
cached = require(addonPath);
|
||||
return cached;
|
||||
} catch (error) {
|
||||
attempts.push(`${addonPath}: ${error?.message || String(error)}`);
|
||||
}
|
||||
} else {
|
||||
attempts.push(`${addonPath}: file not found`);
|
||||
}
|
||||
} else {
|
||||
attempts.push(`${sherpaPlatformPackageName()}: platform package not installed`);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
[
|
||||
`Failed to load sherpa-onnx-node for ${process.platform}-${process.arch}.`,
|
||||
`Node ${process.version} (ABI ${process.versions.modules}).`,
|
||||
'Load attempts:',
|
||||
...attempts.map((line) => `- ${line}`),
|
||||
].join('\n'),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* Sherpa-onnx offline recognizer engine (NeMo transducer / Parakeet) plus a
|
||||
* realtime streaming transcription session that re-decodes the accumulated
|
||||
* segment audio on a throttle to produce live partial transcripts.
|
||||
*
|
||||
* Runs inside the dictation worker process only — never load the native
|
||||
* addon in the main server process.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { existsSync } from 'fs';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
import { loadSherpaOnnxNode } from './sherpa-loader.js';
|
||||
import { pcm16lePeakAbs, pcm16leToFloat32 } from '../audio.js';
|
||||
|
||||
function assertFileExists(filePath, label) {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class SherpaOfflineRecognizerEngine {
|
||||
/**
|
||||
* @param {{ type: 'nemo_transducer' | 'whisper',
|
||||
* encoder: string, decoder: string, joiner?: string, tokens: string,
|
||||
* numThreads?: number }} config
|
||||
*/
|
||||
constructor(config) {
|
||||
assertFileExists(config.encoder, 'offline encoder');
|
||||
assertFileExists(config.decoder, 'offline decoder');
|
||||
if (config.type === 'nemo_transducer') {
|
||||
assertFileExists(config.joiner, 'offline joiner');
|
||||
}
|
||||
assertFileExists(config.tokens, 'tokens');
|
||||
|
||||
const sherpa = loadSherpaOnnxNode();
|
||||
|
||||
const modelConfig =
|
||||
config.type === 'whisper'
|
||||
? {
|
||||
whisper: {
|
||||
encoder: config.encoder,
|
||||
decoder: config.decoder,
|
||||
// Empty language auto-detects for multilingual Whisper exports.
|
||||
language: '',
|
||||
task: 'transcribe',
|
||||
tailPaddings: -1,
|
||||
},
|
||||
tokens: config.tokens,
|
||||
modelType: 'whisper',
|
||||
numThreads: config.numThreads ?? 2,
|
||||
provider: 'cpu',
|
||||
debug: 0,
|
||||
}
|
||||
: {
|
||||
transducer: {
|
||||
encoder: config.encoder,
|
||||
decoder: config.decoder,
|
||||
joiner: config.joiner,
|
||||
},
|
||||
tokens: config.tokens,
|
||||
modelType: 'nemo_transducer',
|
||||
numThreads: config.numThreads ?? 2,
|
||||
provider: 'cpu',
|
||||
debug: 0,
|
||||
};
|
||||
|
||||
const recognizerConfig = {
|
||||
featConfig: {
|
||||
sampleRate: 16000,
|
||||
featureDim: 80,
|
||||
},
|
||||
modelConfig,
|
||||
decodingMethod: 'greedy_search',
|
||||
maxActivePaths: 4,
|
||||
};
|
||||
|
||||
this.recognizer = new sherpa.OfflineRecognizer(recognizerConfig);
|
||||
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
||||
this.sampleRate =
|
||||
typeof sr === 'number' && Number.isFinite(sr) && sr > 0
|
||||
? sr
|
||||
: recognizerConfig.featConfig.sampleRate;
|
||||
}
|
||||
|
||||
createStream() {
|
||||
return this.recognizer.createStream();
|
||||
}
|
||||
|
||||
acceptWaveform(stream, sampleRate, samples) {
|
||||
if (!stream || typeof stream.acceptWaveform !== 'function') {
|
||||
throw new Error('Unexpected sherpa offline stream: missing acceptWaveform()');
|
||||
}
|
||||
// sherpa-onnx-node expects acceptWaveform({ samples, sampleRate });
|
||||
// the WASM build expects acceptWaveform(sampleRate, samples).
|
||||
if (stream.acceptWaveform.length <= 1) {
|
||||
stream.acceptWaveform({ samples, sampleRate });
|
||||
} else {
|
||||
stream.acceptWaveform(sampleRate, samples);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a full PCM16 segment and return its text.
|
||||
* Applies auto-gain when the peak is low so quiet microphones still decode.
|
||||
* @param {Buffer} pcm16
|
||||
* @returns {string}
|
||||
*/
|
||||
decodePcm16(pcm16) {
|
||||
if (pcm16.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const peak = pcm16lePeakAbs(pcm16);
|
||||
const peakFloat = peak / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||
|
||||
const stream = this.createStream();
|
||||
try {
|
||||
const floatSamples = pcm16leToFloat32(pcm16, gain);
|
||||
this.acceptWaveform(stream, this.sampleRate, floatSamples);
|
||||
this.recognizer.decode(stream);
|
||||
const result = this.recognizer.getResult(stream);
|
||||
const text =
|
||||
typeof result === 'object' && result && 'text' in result ? result.text : result;
|
||||
return String(text ?? '').trim();
|
||||
} finally {
|
||||
try {
|
||||
stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
free() {
|
||||
try {
|
||||
this.recognizer?.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming transcription session backed by the offline recognizer.
|
||||
* Accumulates the current segment's PCM and re-decodes it at most every
|
||||
* `minDecodeIntervalMs` to emit non-final partial transcripts; `commit()`
|
||||
* finalizes the segment and starts a new one.
|
||||
*
|
||||
* Implements the StreamingTranscriptionSession contract used by
|
||||
* DictationStreamManager.
|
||||
*/
|
||||
export class SherpaRealtimeTranscriptionSession extends EventEmitter {
|
||||
/**
|
||||
* @param {{ engine: SherpaOfflineRecognizerEngine, minDecodeIntervalMs?: number }} params
|
||||
*/
|
||||
constructor({ engine, minDecodeIntervalMs }) {
|
||||
super();
|
||||
this.engine = engine;
|
||||
this.requiredSampleRate = engine.sampleRate;
|
||||
this.minDecodeIntervalMs = minDecodeIntervalMs ?? 350;
|
||||
this.connected = false;
|
||||
this.currentSegmentId = null;
|
||||
this.previousSegmentId = null;
|
||||
this.lastPartialText = '';
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
this.lastDecodeAt = 0;
|
||||
this.decoding = false;
|
||||
this.pendingDecode = false;
|
||||
}
|
||||
|
||||
async connect() {
|
||||
if (this.connected) {
|
||||
return;
|
||||
}
|
||||
this.currentSegmentId = randomUUID();
|
||||
this.connected = true;
|
||||
}
|
||||
|
||||
appendPcm16(chunk) {
|
||||
if (!this.connected || !this.currentSegmentId) {
|
||||
this.emit('error', new Error('Sherpa realtime session not connected'));
|
||||
return;
|
||||
}
|
||||
this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
|
||||
this.maybeDecode(false).catch((err) => {
|
||||
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
}
|
||||
|
||||
commit() {
|
||||
if (!this.connected || !this.currentSegmentId) {
|
||||
this.emit('error', new Error('Sherpa realtime session not connected'));
|
||||
return;
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
await this.maybeDecode(true);
|
||||
const finalText = this.lastPartialText;
|
||||
const segmentId = this.currentSegmentId;
|
||||
const previousSegmentId = this.previousSegmentId;
|
||||
|
||||
this.emit('committed', { segmentId, previousSegmentId });
|
||||
this.emit('transcript', { segmentId, transcript: finalText, isFinal: true });
|
||||
|
||||
this.previousSegmentId = segmentId;
|
||||
this.currentSegmentId = randomUUID();
|
||||
this.lastPartialText = '';
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
} catch (err) {
|
||||
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
clear() {
|
||||
if (!this.connected) {
|
||||
return;
|
||||
}
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
this.currentSegmentId = randomUUID();
|
||||
this.lastPartialText = '';
|
||||
}
|
||||
|
||||
close() {
|
||||
this.connected = false;
|
||||
this.currentSegmentId = null;
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
}
|
||||
|
||||
async maybeDecode(force) {
|
||||
if (!this.connected || !this.currentSegmentId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.decoding) {
|
||||
this.pendingDecode = true;
|
||||
return;
|
||||
}
|
||||
|
||||
this.decoding = true;
|
||||
try {
|
||||
const decodeStartedAt = Date.now();
|
||||
const text = this.engine.decodePcm16(this.pcm16);
|
||||
this.lastDecodeAt = Date.now();
|
||||
// Adaptive throttle: on slow hardware (or heavy models) re-decoding the
|
||||
// growing segment every 350ms would monopolize the worker. Space partial
|
||||
// decodes to ~1.5x the observed decode time.
|
||||
this.minDecodeIntervalMs = Math.max(350, (this.lastDecodeAt - decodeStartedAt) * 1.5);
|
||||
if (text !== this.lastPartialText) {
|
||||
this.lastPartialText = text;
|
||||
this.emit('transcript', {
|
||||
segmentId: this.currentSegmentId,
|
||||
transcript: text,
|
||||
isFinal: false,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
this.decoding = false;
|
||||
if (this.pendingDecode) {
|
||||
this.pendingDecode = false;
|
||||
await this.maybeDecode(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Sherpa-onnx offline TTS (Kokoro). Runs inside the dictation worker process
|
||||
* only — never load the native addon in the main server process.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { loadSherpaOnnxNode } from './sherpa-loader.js';
|
||||
|
||||
function assertFileExists(filePath, label) {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function float32ToPcm16le(samples) {
|
||||
const out = new Int16Array(samples.length);
|
||||
for (let i = 0; i < samples.length; i += 1) {
|
||||
const clamped = Math.max(-1, Math.min(1, samples[i]));
|
||||
out[i] = Math.round(clamped * 32767);
|
||||
}
|
||||
return Buffer.from(out.buffer, out.byteOffset, out.byteLength);
|
||||
}
|
||||
|
||||
export class SherpaTtsEngine {
|
||||
/**
|
||||
* @param {{ modelDir: string, files: { model: string, voices: string, tokens: string, espeakData: string }, numThreads?: number }} config
|
||||
*/
|
||||
constructor(config) {
|
||||
const modelPath = path.join(config.modelDir, config.files.model);
|
||||
const voicesPath = path.join(config.modelDir, config.files.voices);
|
||||
const tokensPath = path.join(config.modelDir, config.files.tokens);
|
||||
const dataDir = path.join(config.modelDir, config.files.espeakData);
|
||||
|
||||
assertFileExists(modelPath, 'TTS model');
|
||||
assertFileExists(voicesPath, 'TTS voices');
|
||||
assertFileExists(tokensPath, 'TTS tokens');
|
||||
assertFileExists(dataDir, 'TTS espeak-ng dataDir');
|
||||
|
||||
const sherpa = loadSherpaOnnxNode();
|
||||
if (typeof sherpa.OfflineTts !== 'function') {
|
||||
throw new Error('sherpa-onnx-node OfflineTts is unavailable');
|
||||
}
|
||||
|
||||
this.tts = new sherpa.OfflineTts({
|
||||
model: {
|
||||
kokoro: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: 1.0,
|
||||
},
|
||||
},
|
||||
numThreads: config.numThreads ?? 2,
|
||||
provider: 'cpu',
|
||||
maxNumSentences: 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthesize text to PCM16LE.
|
||||
* @param {string} text
|
||||
* @param {{ speakerId?: number, speed?: number }} [options]
|
||||
* @returns {{ pcm16: Buffer, sampleRate: number }}
|
||||
*/
|
||||
synthesize(text, options = {}) {
|
||||
const trimmed = String(text || '').trim();
|
||||
if (!trimmed) {
|
||||
throw new Error('Cannot synthesize empty text');
|
||||
}
|
||||
|
||||
const audio = this.tts.generate({
|
||||
text: trimmed,
|
||||
sid: Number.isInteger(options.speakerId) ? options.speakerId : 0,
|
||||
speed: typeof options.speed === 'number' && options.speed > 0 ? options.speed : 1.0,
|
||||
// Request a copied buffer from sherpa itself: native external-backed
|
||||
// typed arrays are rejected by Electron.
|
||||
enableExternalBuffer: false,
|
||||
});
|
||||
|
||||
let samples = null;
|
||||
if (audio && audio.samples instanceof Float32Array) {
|
||||
samples = Float32Array.from(audio.samples);
|
||||
} else if (audio && Array.isArray(audio.samples)) {
|
||||
samples = Float32Array.from(audio.samples);
|
||||
}
|
||||
if (!samples) {
|
||||
throw new Error('Unexpected sherpa TTS output: missing Float32 samples');
|
||||
}
|
||||
|
||||
const sampleRate =
|
||||
audio && typeof audio.sampleRate === 'number' && audio.sampleRate > 0
|
||||
? audio.sampleRate
|
||||
: typeof this.tts.sampleRate === 'number' && this.tts.sampleRate > 0
|
||||
? this.tts.sampleRate
|
||||
: 24000;
|
||||
|
||||
return { pcm16: float32ToPcm16le(samples), sampleRate };
|
||||
}
|
||||
|
||||
free() {
|
||||
try {
|
||||
this.tts?.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
/**
|
||||
* Client for the dictation local-speech worker process.
|
||||
*
|
||||
* Lazily forks the worker on first use, correlates request/response messages
|
||||
* by requestId, routes session events to per-session EventEmitters, and
|
||||
* shuts the worker down after an idle TTL so the ONNX runtime does not sit
|
||||
* in memory while dictation is unused.
|
||||
*/
|
||||
|
||||
import { fork } from 'child_process';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
import { applySherpaLoaderEnv } from './sherpa-loader.js';
|
||||
|
||||
const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
|
||||
const DEFAULT_IDLE_TTL_MS = 5 * 60 * 1000;
|
||||
const DEFAULT_LOCAL_SAMPLE_RATE = 16000;
|
||||
const STDERR_TAIL_MAX_CHARS = 2000;
|
||||
|
||||
function forkDictationWorker() {
|
||||
const env = { ...process.env };
|
||||
applySherpaLoaderEnv(env);
|
||||
return fork(fileURLToPath(new URL('./worker-process.js', import.meta.url)), [], {
|
||||
env,
|
||||
serialization: 'advanced',
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
windowsHide: true,
|
||||
});
|
||||
}
|
||||
|
||||
export class DictationWorkerClient {
|
||||
/**
|
||||
* @param {{ requestTimeoutMs?: number, idleTtlMs?: number }} [options]
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
||||
this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
|
||||
this.pendingRequests = new Map();
|
||||
this.sessionEmitters = new Map();
|
||||
this.worker = null;
|
||||
this.stderrTail = '';
|
||||
this.inFlightRequests = 0;
|
||||
this.idleTimer = null;
|
||||
this.intentionalCloses = new WeakSet();
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthesize speech in the worker. Returns WAV bytes.
|
||||
* @param {{ modelsDir: string, modelId: string, text: string, speakerId?: number, speed?: number }} params
|
||||
* @returns {Promise<{ audio: Buffer, format: string }>}
|
||||
*/
|
||||
async synthesizeSpeech(params) {
|
||||
// Long texts on slow hardware can exceed the default request timeout.
|
||||
const result = await this.sendRequest(
|
||||
{ type: 'tts.synthesize', ...params },
|
||||
{ timeoutMs: 120000 },
|
||||
);
|
||||
return {
|
||||
audio: Buffer.isBuffer(result.audio) ? result.audio : Buffer.from(result.audio),
|
||||
format: result.format || 'audio/wav',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a streaming STT session in the worker.
|
||||
* @param {{ modelsDir: string, modelId: string }} params
|
||||
* @param {EventEmitter} emitter receives 'committed' | 'transcript' | 'error'
|
||||
* @returns {Promise<{ sessionId: string, requiredSampleRate: number }>}
|
||||
*/
|
||||
async createSession({ modelsDir, modelId }, emitter) {
|
||||
const sessionId = randomUUID();
|
||||
this.sessionEmitters.set(sessionId, emitter);
|
||||
try {
|
||||
const result = await this.sendRequest({
|
||||
type: 'session.create',
|
||||
sessionId,
|
||||
modelsDir,
|
||||
modelId,
|
||||
});
|
||||
return { sessionId, requiredSampleRate: result?.requiredSampleRate ?? DEFAULT_LOCAL_SAMPLE_RATE };
|
||||
} catch (err) {
|
||||
this.sessionEmitters.delete(sessionId);
|
||||
this.scheduleIdleShutdownIfReady();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
appendSessionAudio(sessionId, audio) {
|
||||
void this.sendRequest({ type: 'session.append', sessionId, audio }).catch((err) => {
|
||||
this.emitSessionError(sessionId, err);
|
||||
});
|
||||
}
|
||||
|
||||
commitSession(sessionId) {
|
||||
void this.sendRequest({ type: 'session.commit', sessionId }).catch((err) => {
|
||||
this.emitSessionError(sessionId, err);
|
||||
});
|
||||
}
|
||||
|
||||
clearSession(sessionId) {
|
||||
void this.sendRequest({ type: 'session.clear', sessionId }).catch((err) => {
|
||||
this.emitSessionError(sessionId, err);
|
||||
});
|
||||
}
|
||||
|
||||
closeSession(sessionId) {
|
||||
this.sessionEmitters.delete(sessionId);
|
||||
void this.sendRequest({ type: 'session.close', sessionId }).catch(() => {
|
||||
// Closing is best-effort; the parent already dropped the session.
|
||||
});
|
||||
this.scheduleIdleShutdownIfReady();
|
||||
}
|
||||
|
||||
shutdown() {
|
||||
this.clearIdleTimer();
|
||||
this.rejectAllPending(new Error('Dictation worker shut down'));
|
||||
this.sessionEmitters.clear();
|
||||
const worker = this.worker;
|
||||
this.worker = null;
|
||||
if (worker && !worker.killed) {
|
||||
this.intentionalCloses.add(worker);
|
||||
try {
|
||||
worker.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
worker.kill();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sendRequest(input, options = {}) {
|
||||
const worker = this.ensureWorker();
|
||||
const requestId = randomUUID();
|
||||
const message = { ...input, requestId };
|
||||
this.inFlightRequests += 1;
|
||||
this.clearIdleTimer();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.pendingRequests.delete(requestId);
|
||||
this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
|
||||
this.scheduleIdleShutdownIfReady();
|
||||
reject(new Error(`Dictation worker request timed out: ${input.type}`));
|
||||
}, options.timeoutMs ?? this.requestTimeoutMs);
|
||||
|
||||
this.pendingRequests.set(requestId, { resolve, reject, timeout });
|
||||
|
||||
worker.send(message, (error) => {
|
||||
if (!error) {
|
||||
return;
|
||||
}
|
||||
const pending = this.pendingRequests.get(requestId);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(pending.timeout);
|
||||
this.pendingRequests.delete(requestId);
|
||||
this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
|
||||
this.scheduleIdleShutdownIfReady();
|
||||
pending.reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
ensureWorker() {
|
||||
if (this.worker && !this.worker.killed && this.worker.connected) {
|
||||
return this.worker;
|
||||
}
|
||||
const worker = forkDictationWorker();
|
||||
this.worker = worker;
|
||||
this.stderrTail = '';
|
||||
worker.stderr?.on('data', (chunk) => {
|
||||
const text = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk);
|
||||
this.stderrTail = (this.stderrTail + text).slice(-STDERR_TAIL_MAX_CHARS);
|
||||
});
|
||||
worker.on('message', (message) => this.handleWorkerMessage(message));
|
||||
worker.on('close', (code, signal) => this.handleWorkerExit(worker, code, signal));
|
||||
return worker;
|
||||
}
|
||||
|
||||
handleWorkerMessage(message) {
|
||||
if (message?.type === 'response') {
|
||||
const pending = this.pendingRequests.get(message.requestId);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(pending.timeout);
|
||||
this.pendingRequests.delete(message.requestId);
|
||||
this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
|
||||
this.scheduleIdleShutdownIfReady();
|
||||
if (message.ok) {
|
||||
pending.resolve(message.result);
|
||||
} else {
|
||||
pending.reject(new Error(message.error || 'Dictation worker request failed'));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const emitter = this.sessionEmitters.get(message?.sessionId);
|
||||
if (!emitter) {
|
||||
return;
|
||||
}
|
||||
switch (message.type) {
|
||||
case 'session.committed':
|
||||
emitter.emit('committed', message.payload);
|
||||
return;
|
||||
case 'session.transcript':
|
||||
emitter.emit('transcript', message.payload);
|
||||
return;
|
||||
case 'session.error':
|
||||
emitter.emit('error', new Error(message.error));
|
||||
return;
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
handleWorkerExit(worker, code, signal) {
|
||||
const wasCurrentWorker = this.worker === worker;
|
||||
const wasIntentional = this.intentionalCloses.has(worker);
|
||||
this.intentionalCloses.delete(worker);
|
||||
if (!wasCurrentWorker || wasIntentional) {
|
||||
if (wasCurrentWorker) {
|
||||
this.worker = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const stderr = this.stderrTail.trim();
|
||||
const error = new Error(
|
||||
`Dictation worker exited (code ${code ?? 'null'}${signal ? `, signal ${signal}` : ''}).` +
|
||||
(stderr ? ` Last stderr: ${stderr.slice(-500)}` : ''),
|
||||
);
|
||||
|
||||
this.worker = null;
|
||||
this.clearIdleTimer();
|
||||
this.rejectAllPending(error);
|
||||
for (const emitter of this.sessionEmitters.values()) {
|
||||
if (emitter.listenerCount('error') > 0) {
|
||||
emitter.emit('error', error);
|
||||
}
|
||||
}
|
||||
this.sessionEmitters.clear();
|
||||
this.inFlightRequests = 0;
|
||||
}
|
||||
|
||||
rejectAllPending(error) {
|
||||
for (const [requestId, pending] of this.pendingRequests) {
|
||||
clearTimeout(pending.timeout);
|
||||
pending.reject(error);
|
||||
this.pendingRequests.delete(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
emitSessionError(sessionId, error) {
|
||||
const emitter = this.sessionEmitters.get(sessionId);
|
||||
if (emitter && emitter.listenerCount('error') > 0) {
|
||||
emitter.emit('error', error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
|
||||
scheduleIdleShutdownIfReady() {
|
||||
if (!this.worker || this.inFlightRequests > 0 || this.sessionEmitters.size > 0) {
|
||||
return;
|
||||
}
|
||||
this.clearIdleTimer();
|
||||
this.idleTimer = setTimeout(() => {
|
||||
if (this.inFlightRequests === 0 && this.sessionEmitters.size === 0) {
|
||||
this.shutdown();
|
||||
}
|
||||
}, this.idleTtlMs);
|
||||
}
|
||||
|
||||
clearIdleTimer() {
|
||||
if (this.idleTimer) {
|
||||
clearTimeout(this.idleTimer);
|
||||
this.idleTimer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* StreamingTranscriptionSession backed by the worker process.
|
||||
* Matches the session contract consumed by DictationStreamManager.
|
||||
*/
|
||||
export class WorkerBackedTranscriptionSession extends EventEmitter {
|
||||
/**
|
||||
* @param {DictationWorkerClient} client
|
||||
* @param {{ modelsDir: string, modelId: string }} modelConfig
|
||||
*/
|
||||
constructor(client, modelConfig) {
|
||||
super();
|
||||
this.client = client;
|
||||
this.modelConfig = modelConfig;
|
||||
this.requiredSampleRate = DEFAULT_LOCAL_SAMPLE_RATE;
|
||||
this.connectedSessionId = null;
|
||||
this.connecting = null;
|
||||
}
|
||||
|
||||
async connect() {
|
||||
if (this.connectedSessionId) {
|
||||
return;
|
||||
}
|
||||
if (!this.connecting) {
|
||||
this.connecting = (async () => {
|
||||
try {
|
||||
const result = await this.client.createSession(this.modelConfig, this);
|
||||
this.connectedSessionId = result.sessionId;
|
||||
this.requiredSampleRate = result.requiredSampleRate;
|
||||
} finally {
|
||||
this.connecting = null;
|
||||
}
|
||||
})();
|
||||
}
|
||||
await this.connecting;
|
||||
}
|
||||
|
||||
appendPcm16(pcm16le) {
|
||||
if (!this.connectedSessionId) {
|
||||
this.emit('error', new Error('Local STT session not connected'));
|
||||
return;
|
||||
}
|
||||
this.client.appendSessionAudio(this.connectedSessionId, pcm16le);
|
||||
}
|
||||
|
||||
commit() {
|
||||
if (!this.connectedSessionId) {
|
||||
this.emit('error', new Error('Local STT session not connected'));
|
||||
return;
|
||||
}
|
||||
this.client.commitSession(this.connectedSessionId);
|
||||
}
|
||||
|
||||
clear() {
|
||||
if (this.connectedSessionId) {
|
||||
this.client.clearSession(this.connectedSessionId);
|
||||
}
|
||||
}
|
||||
|
||||
close() {
|
||||
const sessionId = this.connectedSessionId;
|
||||
this.connectedSessionId = null;
|
||||
if (sessionId) {
|
||||
this.client.closeSession(sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* Dictation local-speech worker process.
|
||||
*
|
||||
* Hosts the sherpa-onnx native inference (Parakeet STT) in a separate process
|
||||
* so ONNX decoding never blocks the main OpenChamber server. Communicates
|
||||
* with the parent over child_process IPC (advanced serialization, so Buffers
|
||||
* survive the trip as Uint8Array).
|
||||
*
|
||||
* Request/response protocol (parent -> worker):
|
||||
* { type: 'session.create', requestId, sessionId, modelsDir, modelId }
|
||||
* { type: 'session.append', requestId, sessionId, audio }
|
||||
* { type: 'session.commit' | 'session.clear' | 'session.close', requestId, sessionId }
|
||||
* Worker -> parent:
|
||||
* { type: 'response', requestId, ok, result?, error? }
|
||||
* { type: 'session.committed' | 'session.transcript' | 'session.error', sessionId, ... }
|
||||
*/
|
||||
|
||||
import {
|
||||
SherpaOfflineRecognizerEngine,
|
||||
SherpaRealtimeTranscriptionSession,
|
||||
} from './sherpa-recognizer.js';
|
||||
import { SherpaTtsEngine } from './sherpa-tts.js';
|
||||
import { getLocalSttModelDir, getLocalSttModelSpec } from './model-catalog.js';
|
||||
import { pcm16ToWav } from '../audio.js';
|
||||
import path from 'path';
|
||||
|
||||
process.title = 'OpenChamber Dictation';
|
||||
|
||||
const engines = new Map();
|
||||
const ttsEngines = new Map();
|
||||
const sessions = new Map();
|
||||
let ipcClosing = false;
|
||||
|
||||
function sendToParent(message) {
|
||||
if (ipcClosing || !process.connected || !process.send) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
process.send(message, (error) => {
|
||||
if (error) {
|
||||
ipcClosing = true;
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
ipcClosing = true;
|
||||
}
|
||||
}
|
||||
|
||||
function sendOk(requestId, result) {
|
||||
sendToParent({ type: 'response', requestId, ok: true, ...(result !== undefined ? { result } : {}) });
|
||||
}
|
||||
|
||||
function getEngine(modelsDir, modelId) {
|
||||
const key = `${modelsDir}:${modelId}`;
|
||||
const existing = engines.get(key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const modelDir = getLocalSttModelDir(modelsDir, modelId);
|
||||
const spec = getLocalSttModelSpec(modelId);
|
||||
const created = new SherpaOfflineRecognizerEngine({
|
||||
type: spec.type,
|
||||
encoder: path.join(modelDir, spec.files.encoder),
|
||||
decoder: path.join(modelDir, spec.files.decoder),
|
||||
...(spec.files.joiner ? { joiner: path.join(modelDir, spec.files.joiner) } : {}),
|
||||
tokens: path.join(modelDir, spec.files.tokens),
|
||||
numThreads: 2,
|
||||
});
|
||||
engines.set(key, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
function cleanupSession(sessionId) {
|
||||
const session = sessions.get(sessionId);
|
||||
sessions.delete(sessionId);
|
||||
try {
|
||||
session?.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
function toBuffer(audio) {
|
||||
if (Buffer.isBuffer(audio)) {
|
||||
return audio;
|
||||
}
|
||||
if (audio instanceof Uint8Array) {
|
||||
return Buffer.from(audio.buffer, audio.byteOffset, audio.byteLength);
|
||||
}
|
||||
if (audio && typeof audio === 'object' && audio.type === 'Buffer' && Array.isArray(audio.data)) {
|
||||
return Buffer.from(audio.data);
|
||||
}
|
||||
throw new Error('Unsupported audio payload in dictation worker');
|
||||
}
|
||||
|
||||
function getTtsEngine(modelsDir, modelId) {
|
||||
const key = `${modelsDir}:${modelId}`;
|
||||
const existing = ttsEngines.get(key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const spec = getLocalSttModelSpec(modelId);
|
||||
const created = new SherpaTtsEngine({
|
||||
modelDir: getLocalSttModelDir(modelsDir, modelId),
|
||||
files: spec.files,
|
||||
numThreads: 2,
|
||||
});
|
||||
ttsEngines.set(key, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
async function handleRequest(message) {
|
||||
switch (message.type) {
|
||||
case 'tts.synthesize': {
|
||||
const engine = getTtsEngine(message.modelsDir, message.modelId);
|
||||
const { pcm16, sampleRate } = engine.synthesize(message.text, {
|
||||
speakerId: message.speakerId,
|
||||
speed: message.speed,
|
||||
});
|
||||
sendOk(message.requestId, {
|
||||
audio: pcm16ToWav(pcm16, sampleRate),
|
||||
format: 'audio/wav',
|
||||
});
|
||||
return;
|
||||
}
|
||||
case 'session.create': {
|
||||
cleanupSession(message.sessionId);
|
||||
const engine = getEngine(message.modelsDir, message.modelId);
|
||||
const session = new SherpaRealtimeTranscriptionSession({ engine });
|
||||
session.on('committed', (payload) => {
|
||||
sendToParent({ type: 'session.committed', sessionId: message.sessionId, payload });
|
||||
});
|
||||
session.on('transcript', (payload) => {
|
||||
sendToParent({ type: 'session.transcript', sessionId: message.sessionId, payload });
|
||||
});
|
||||
session.on('error', (err) => {
|
||||
sendToParent({
|
||||
type: 'session.error',
|
||||
sessionId: message.sessionId,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
});
|
||||
await session.connect();
|
||||
sessions.set(message.sessionId, session);
|
||||
sendOk(message.requestId, { requiredSampleRate: session.requiredSampleRate });
|
||||
return;
|
||||
}
|
||||
case 'session.append': {
|
||||
sessions.get(message.sessionId)?.appendPcm16(toBuffer(message.audio));
|
||||
sendOk(message.requestId);
|
||||
return;
|
||||
}
|
||||
case 'session.commit': {
|
||||
sessions.get(message.sessionId)?.commit();
|
||||
sendOk(message.requestId);
|
||||
return;
|
||||
}
|
||||
case 'session.clear': {
|
||||
sessions.get(message.sessionId)?.clear();
|
||||
sendOk(message.requestId);
|
||||
return;
|
||||
}
|
||||
case 'session.close': {
|
||||
cleanupSession(message.sessionId);
|
||||
sendOk(message.requestId);
|
||||
return;
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Unknown dictation worker request: ${message?.type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process.on('message', (message) => {
|
||||
void handleRequest(message).catch((error) => {
|
||||
sendToParent({
|
||||
type: 'response',
|
||||
requestId: message?.requestId,
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Dictation worker request failed',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
process.once('disconnect', () => {
|
||||
ipcClosing = true;
|
||||
for (const sessionId of Array.from(sessions.keys())) {
|
||||
cleanupSession(sessionId);
|
||||
}
|
||||
for (const engine of engines.values()) {
|
||||
engine.free();
|
||||
}
|
||||
for (const tts of ttsEngines.values()) {
|
||||
tts.free();
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
Reference in New Issue
Block a user