feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)
Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
This commit is contained in:
Bohdan Triapitsyn
committed by
GitHub
GitHub
parent
3f5151d424
commit
de1b85ac56
@@ -113,7 +113,10 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
|
||||
localStorage.removeItem('opencode-update-toast-dismissed-version');
|
||||
}
|
||||
}
|
||||
if (settings.sttProvider === 'browser' || settings.sttProvider === 'server') {
|
||||
if (typeof settings.dictationEnabled === 'boolean') {
|
||||
localStorage.setItem('dictationEnabled', String(settings.dictationEnabled));
|
||||
}
|
||||
if (settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') {
|
||||
localStorage.setItem('sttProvider', settings.sttProvider);
|
||||
}
|
||||
if (typeof settings.sttServerUrl === 'string') {
|
||||
@@ -122,18 +125,12 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
|
||||
if (typeof settings.sttModel === 'string') {
|
||||
localStorage.setItem('sttModel', settings.sttModel);
|
||||
}
|
||||
if (typeof settings.sttLocalModel === 'string') {
|
||||
localStorage.setItem('sttLocalModel', settings.sttLocalModel);
|
||||
}
|
||||
if (typeof settings.sttLanguage === 'string') {
|
||||
localStorage.setItem('sttLanguage', settings.sttLanguage);
|
||||
}
|
||||
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb)) {
|
||||
localStorage.setItem('sttSilenceThresholdDb', String(settings.sttSilenceThresholdDb));
|
||||
}
|
||||
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs)) {
|
||||
localStorage.setItem('sttSilenceHoldMs', String(settings.sttSilenceHoldMs));
|
||||
}
|
||||
if (typeof settings.sttTranscribeOnStop === 'boolean') {
|
||||
localStorage.setItem('sttTranscribeOnStop', String(settings.sttTranscribeOnStop));
|
||||
}
|
||||
};
|
||||
|
||||
const dispatchSettingsSynced = (settings: DesktopSettings): void => {
|
||||
@@ -614,7 +611,10 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
|
||||
}
|
||||
if (configStoreApi && configStore) {
|
||||
const nextConfigState: Partial<typeof configStore> = {};
|
||||
if ((settings.sttProvider === 'browser' || settings.sttProvider === 'server') && settings.sttProvider !== configStore.sttProvider) {
|
||||
if (typeof settings.dictationEnabled === 'boolean' && settings.dictationEnabled !== configStore.dictationEnabled) {
|
||||
nextConfigState.dictationEnabled = settings.dictationEnabled;
|
||||
}
|
||||
if ((settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') && settings.sttProvider !== configStore.sttProvider) {
|
||||
nextConfigState.sttProvider = settings.sttProvider;
|
||||
}
|
||||
if (typeof settings.sttServerUrl === 'string' && settings.sttServerUrl !== configStore.sttServerUrl) {
|
||||
@@ -623,18 +623,12 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
|
||||
if (typeof settings.sttModel === 'string' && settings.sttModel !== configStore.sttModel) {
|
||||
nextConfigState.sttModel = settings.sttModel;
|
||||
}
|
||||
if (typeof settings.sttLocalModel === 'string' && settings.sttLocalModel !== configStore.sttLocalModel) {
|
||||
nextConfigState.sttLocalModel = settings.sttLocalModel;
|
||||
}
|
||||
if (typeof settings.sttLanguage === 'string' && settings.sttLanguage !== configStore.sttLanguage) {
|
||||
nextConfigState.sttLanguage = settings.sttLanguage;
|
||||
}
|
||||
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb) && settings.sttSilenceThresholdDb !== configStore.sttSilenceThresholdDb) {
|
||||
nextConfigState.sttSilenceThresholdDb = settings.sttSilenceThresholdDb;
|
||||
}
|
||||
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs) && settings.sttSilenceHoldMs !== configStore.sttSilenceHoldMs) {
|
||||
nextConfigState.sttSilenceHoldMs = settings.sttSilenceHoldMs;
|
||||
}
|
||||
if (typeof settings.sttTranscribeOnStop === 'boolean' && settings.sttTranscribeOnStop !== configStore.sttTranscribeOnStop) {
|
||||
nextConfigState.sttTranscribeOnStop = settings.sttTranscribeOnStop;
|
||||
}
|
||||
if (Object.keys(nextConfigState).length > 0) {
|
||||
configStoreApi.setState(nextConfigState);
|
||||
}
|
||||
@@ -1200,8 +1194,16 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
|
||||
if (typeof candidate.responseStyleCustomInstructions === 'string') {
|
||||
result.responseStyleCustomInstructions = candidate.responseStyleCustomInstructions;
|
||||
}
|
||||
if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'server') {
|
||||
if (typeof candidate.dictationEnabled === 'boolean') {
|
||||
result.dictationEnabled = candidate.dictationEnabled;
|
||||
}
|
||||
if (candidate.sttProvider === 'local' || candidate.sttProvider === 'openai-compatible') {
|
||||
result.sttProvider = candidate.sttProvider;
|
||||
} else if (candidate.sttProvider === 'server') {
|
||||
// Legacy provider migration: 'server' was the OpenAI-compatible endpoint.
|
||||
result.sttProvider = 'openai-compatible';
|
||||
} else if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'wasm') {
|
||||
result.sttProvider = 'local';
|
||||
}
|
||||
if (typeof candidate.sttServerUrl === 'string') {
|
||||
result.sttServerUrl = candidate.sttServerUrl.trim();
|
||||
@@ -1209,18 +1211,12 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
|
||||
if (typeof candidate.sttModel === 'string') {
|
||||
result.sttModel = candidate.sttModel.trim();
|
||||
}
|
||||
if (typeof candidate.sttLocalModel === 'string') {
|
||||
result.sttLocalModel = candidate.sttLocalModel.trim();
|
||||
}
|
||||
if (typeof candidate.sttLanguage === 'string') {
|
||||
result.sttLanguage = candidate.sttLanguage.trim();
|
||||
}
|
||||
if (typeof candidate.sttSilenceThresholdDb === 'number' && Number.isFinite(candidate.sttSilenceThresholdDb)) {
|
||||
result.sttSilenceThresholdDb = candidate.sttSilenceThresholdDb;
|
||||
}
|
||||
if (typeof candidate.sttSilenceHoldMs === 'number' && Number.isFinite(candidate.sttSilenceHoldMs)) {
|
||||
result.sttSilenceHoldMs = candidate.sttSilenceHoldMs;
|
||||
}
|
||||
if (typeof candidate.sttTranscribeOnStop === 'boolean') {
|
||||
result.sttTranscribeOnStop = candidate.sttTranscribeOnStop;
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user