Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
137 lines
4.0 KiB
JavaScript
137 lines
4.0 KiB
JavaScript
/**
|
|
* Loader for the sherpa-onnx-node native addon.
|
|
*
|
|
* sherpa-onnx-node ships its native addon and shared libraries in a
|
|
* platform-specific package (e.g. sherpa-onnx-darwin-arm64). The shared
|
|
* libraries must be findable via the platform's dynamic-loader search path,
|
|
* so the loader prepends the platform package directory to LD_LIBRARY_PATH /
|
|
* DYLD_LIBRARY_PATH / PATH before requiring the addon.
|
|
*/
|
|
|
|
import { createRequire } from 'module';
|
|
import path from 'path';
|
|
import { existsSync } from 'fs';
|
|
|
|
const require = createRequire(import.meta.url);
|
|
|
|
let cached = null;
|
|
|
|
function sherpaPlatformPackageName(platform = process.platform, arch = process.arch) {
|
|
const normalizedPlatform = platform === 'win32' ? 'win' : platform;
|
|
return `sherpa-onnx-${normalizedPlatform}-${arch}`;
|
|
}
|
|
|
|
function sherpaLoaderEnvKey(platform = process.platform) {
|
|
if (platform === 'linux') {
|
|
return 'LD_LIBRARY_PATH';
|
|
}
|
|
if (platform === 'darwin') {
|
|
return 'DYLD_LIBRARY_PATH';
|
|
}
|
|
if (platform === 'win32') {
|
|
return 'PATH';
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function prependEnvPath(existing, value) {
|
|
const parts = String(existing ?? '').split(path.delimiter).filter(Boolean);
|
|
if (parts.includes(value)) {
|
|
return parts.join(path.delimiter);
|
|
}
|
|
return [value, ...parts].join(path.delimiter);
|
|
}
|
|
|
|
/**
|
|
* Case-insensitive env key lookup: on Windows `{...process.env}` yields a
|
|
* plain object where PATH may be stored as `Path`. Using a hardcoded 'PATH'
|
|
* would create a duplicate key and break the child process PATH.
|
|
*/
|
|
function findEnvKey(env, key) {
|
|
const lower = key.toLowerCase();
|
|
for (const k of Object.keys(env)) {
|
|
if (k.toLowerCase() === lower) {
|
|
return k;
|
|
}
|
|
}
|
|
return key;
|
|
}
|
|
|
|
function resolveSherpaLibDir(platform = process.platform, arch = process.arch) {
|
|
const packageName = sherpaPlatformPackageName(platform, arch);
|
|
try {
|
|
const pkgJson = require.resolve(`${packageName}/package.json`);
|
|
// Electron packages node_modules inside app.asar, but native addons and
|
|
// their shared libraries are extracted to app.asar.unpacked. The dynamic
|
|
// loader (dlopen/DYLD/LD) cannot read from the asar archive, so point the
|
|
// search path at the unpacked copy.
|
|
const dir = path.dirname(pkgJson);
|
|
const unpacked = dir.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`);
|
|
return existsSync(unpacked) ? unpacked : dir;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Prepend the sherpa platform package dir to the loader search path env var.
|
|
* Mutates the provided env object.
|
|
* @param {NodeJS.ProcessEnv} env
|
|
*/
|
|
export function applySherpaLoaderEnv(env) {
|
|
const key = sherpaLoaderEnvKey();
|
|
const libDir = resolveSherpaLibDir();
|
|
if (!key || !libDir) {
|
|
return { key: null, libDir: null };
|
|
}
|
|
const actualKey = findEnvKey(env, key);
|
|
env[actualKey] = prependEnvPath(env[actualKey], libDir);
|
|
return { key, libDir };
|
|
}
|
|
|
|
/**
|
|
* Load the sherpa-onnx-node module, trying the upstream entry first and then
|
|
* the platform addon directly.
|
|
*/
|
|
export function loadSherpaOnnxNode() {
|
|
if (cached) {
|
|
return cached;
|
|
}
|
|
|
|
const attempts = [];
|
|
|
|
try {
|
|
cached = require('sherpa-onnx-node');
|
|
return cached;
|
|
} catch (error) {
|
|
attempts.push(`sherpa-onnx-node: ${error?.message || String(error)}`);
|
|
}
|
|
|
|
const libDir = resolveSherpaLibDir();
|
|
if (libDir) {
|
|
applySherpaLoaderEnv(process.env);
|
|
const addonPath = path.join(libDir, 'sherpa-onnx.node');
|
|
if (existsSync(addonPath)) {
|
|
try {
|
|
cached = require(addonPath);
|
|
return cached;
|
|
} catch (error) {
|
|
attempts.push(`${addonPath}: ${error?.message || String(error)}`);
|
|
}
|
|
} else {
|
|
attempts.push(`${addonPath}: file not found`);
|
|
}
|
|
} else {
|
|
attempts.push(`${sherpaPlatformPackageName()}: platform package not installed`);
|
|
}
|
|
|
|
throw new Error(
|
|
[
|
|
`Failed to load sherpa-onnx-node for ${process.platform}-${process.arch}.`,
|
|
`Node ${process.version} (ABI ${process.versions.modules}).`,
|
|
'Load attempts:',
|
|
...attempts.map((line) => `- ${line}`),
|
|
].join('\n'),
|
|
);
|
|
}
|