fix: voice input in Electron - local Whisper STT + network error handling

- Add local Whisper STT via Transformers.js with Web Worker (no UI freeze)
- Default sttProvider to 'local' in Electron (browser STT unavailable)
- Fix infinite toast loop: stop auto-restart on network errors
- Add retry limit with exponential backoff for transient STT errors
- Append voice transcript to input field (append-inline), not replace
- Add model catalog with download/load button in Voice Settings
This commit is contained in:
Bohdan Triapitsyn
2026-05-14 01:19:52 +03:00
parent ceb5bbd5dd
commit ef85c63336
18 changed files with 1114 additions and 89 deletions
@@ -17,6 +17,8 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip
import { Icon } from "@/components/icon/Icon";
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
import { audioStreamService } from '@/lib/voice/audioStreamService';
import { wasmSttService, WASM_MODELS } from '@/lib/voice/wasmSttService';
import type { WasmModelStatus } from '@/lib/voice/wasmSttService';
import { cn } from '@/lib/utils';
import { useI18n } from '@/lib/i18n';
const LANGUAGE_OPTIONS = [
@@ -32,6 +34,82 @@ const LANGUAGE_OPTIONS = [
{ value: 'uk-UA', label: 'Українська' },
];
const WasmModelStatusIndicator = ({ modelId }: { modelId: string }) => {
const { t } = useI18n();
const [status, setStatus] = useState<WasmModelStatus>(wasmSttService.getModelStatus());
const [loading, setLoading] = useState(false);
useEffect(() => {
const handler = (s: WasmModelStatus) => setStatus(s);
wasmSttService.onModelStatusChange = handler;
return () => { wasmSttService.onModelStatusChange = null; };
}, []);
const currentModelId = wasmSttService.getCurrentModelId();
const isLoadingOrDownloading = status.state === 'downloading' || status.state === 'loading';
// Reset local loading state when model finishes loading or errors
useEffect(() => {
if (status.state !== 'downloading' && status.state !== 'loading') {
setLoading(false);
}
}, [status.state]);
const handleDownload = async () => {
setLoading(true);
try {
await wasmSttService.loadModel(modelId);
} catch {
// Error is shown via status indicator
}
};
if (status.state === 'ready' && currentModelId === modelId) {
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-green-600 dark:text-green-400">
{t('settings.voice.page.stt.wasmLoaded')}
</span>
</div>
);
}
if (isLoadingOrDownloading) {
const progress = status.state === 'downloading' ? Math.round(status.progress) : undefined;
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-muted-foreground">
{status.state === 'downloading' ? t('settings.voice.page.stt.wasmDownloading') : t('settings.voice.page.stt.wasmLoading')}
{progress !== undefined ? ` (${progress}%)` : ''}
</span>
</div>
);
}
if (status.state === 'error') {
// Partial download (cached progress): show retry button.
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-destructive">{status.error}</span>
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
{t('settings.voice.page.stt.wasmRetry')}
</Button>
</div>
);
}
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-muted-foreground">
{t('settings.voice.page.stt.wasmNotLoaded')}
</span>
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
{t('settings.voice.page.stt.wasmDownload')}
</Button>
</div>
);
};
const OPENAI_VOICE_OPTIONS = [
{ value: 'alloy', label: 'Alloy' },
{ value: 'ash', label: 'Ash' },
@@ -86,6 +164,8 @@ export const VoiceSettings: React.FC = () => {
const setSttServerUrl = useConfigStore((state) => state.setSttServerUrl);
const sttModel = useConfigStore((state) => state.sttModel);
const setSttModel = useConfigStore((state) => state.setSttModel);
const wasmSttModel = useConfigStore((state) => state.wasmSttModel);
const setWasmSttModel = useConfigStore((state) => state.setWasmSttModel);
const sttLanguage = useConfigStore((state) => state.sttLanguage);
const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
@@ -647,12 +727,12 @@ export const VoiceSettings: React.FC = () => {
{voiceProvider === 'browser' && filteredBrowserVoices.length > 0 && (
<>
<Select value={browserVoice || '__auto__'} onValueChange={(value) => setBrowserVoice(value === '__auto__' ? '' : value)}>
<Select value={browserVoice || '$auto'} onValueChange={(value) => setBrowserVoice(value === '$auto' ? '' : value)}>
<SelectTrigger className="w-fit max-w-[200px]">
<SelectValue placeholder={t('settings.voice.page.field.auto')} />
</SelectTrigger>
<SelectContent>
<SelectItem value="__auto__">{t('settings.voice.page.field.auto')}</SelectItem>
<SelectItem value="$auto">{t('settings.voice.page.field.auto')}</SelectItem>
{filteredBrowserVoices.map((v) => (
<SelectItem key={v.name} value={v.name}>{v.name} ({v.lang})</SelectItem>
))}
@@ -765,6 +845,15 @@ export const VoiceSettings: React.FC = () => {
>
{t('settings.voice.page.provider.server')}
</Button>
<Button
variant="chip"
size="xs"
aria-pressed={sttProvider === 'wasm'}
onClick={() => setSttProvider('wasm')}
className="!font-normal"
>
{t('settings.voice.page.provider.wasm')}
</Button>
</div>
</div>
</div>
@@ -860,6 +949,38 @@ export const VoiceSettings: React.FC = () => {
</div>
</div>
)}
{sttProvider === 'wasm' && (
<div className="py-1.5 space-y-2">
<div>
<span className="typography-ui-label text-muted-foreground">
{t('settings.voice.page.stt.wasmModel')}
</span>
<Select value={wasmSttModel} onValueChange={setWasmSttModel}>
<SelectTrigger className="mt-0.5">
<SelectValue placeholder={t('settings.voice.page.stt.wasmModel')} />
</SelectTrigger>
<SelectContent>
{WASM_MODELS.map((m) => (
<SelectItem key={m.id} value={m.id}>
<div className="flex flex-col">
<span>{m.name}</span>
<span className="typography-ui-compact text-muted-foreground">
{m.size} · {m.languages}
</span>
</div>
</SelectItem>
))}
</SelectContent>
</Select>
<p className="typography-ui-compact text-muted-foreground mt-0.5">
{WASM_MODELS.find((m) => m.id === wasmSttModel)?.description}
</p>
</div>
{/* Model status indicator */}
<WasmModelStatusIndicator modelId={wasmSttModel} />
</div>
)}
</section>
</div>
)}
@@ -113,7 +113,9 @@ export function BrowserVoiceButton() {
const isIdle = status === 'idle';
const isSpeaking = status === 'speaking';
const canTranscribeOnStop = sttProvider === 'server' && sttTranscribeOnStop;
// WASM STT always needs finishVoiceInput to flush the recorder and transcribe.
// Server STT uses it when sttTranscribeOnStop is enabled.
const canTranscribeOnStop = sttProvider === 'wasm' || (sttProvider === 'server' && sttTranscribeOnStop);
const isListeningWithTranscribeOnStop = status === 'listening' && canTranscribeOnStop;
// Show toast notification when voice error occurs
+154 -66
View File
@@ -28,6 +28,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
import { audioStreamService } from '@/lib/voice/audioStreamService';
import { wasmSttService } from '@/lib/voice/wasmSttService';
import { useSessionUIStore } from '@/sync/session-ui-store';
import { useInputStore } from '@/sync/input-store';
import { getSyncMessages, getSyncParts } from '@/sync/sync-refs';
@@ -127,6 +128,9 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
const pendingFinalTranscriptRef = useRef('');
const finalTranscriptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const deviceChangeRestartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const consecutiveRecoveryRetriesRef = useRef(0);
const recoveryTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const isFinalizingRef = useRef(false);
// Store access
const currentSessionId = useSessionUIStore((s) => s.currentSessionId);
@@ -157,13 +161,16 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
const sttProvider = useConfigStore((state) => state.sttProvider);
const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
const sttModel = useConfigStore((state) => state.sttModel);
const wasmSttModel = useConfigStore((state) => state.wasmSttModel);
const sttLanguage = useConfigStore((state) => state.sttLanguage);
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
const isSupported = sttProvider === 'server'
? audioStreamService.isSupported()
: browserVoiceService.isSupported();
: sttProvider === 'wasm'
? wasmSttService.isSupported()
: browserVoiceService.isSupported();
// Server TTS for mobile (bypasses Safari audio restrictions)
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS({
@@ -287,6 +294,20 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
const handleSpeechErrorRef = useRef<((errorMsg: string) => void) | null>(null);
const handleSpeechResultRef = useRef<((text: string, isFinal: boolean) => Promise<void>) | null>(null);
// Start STT via the currently-selected provider.
// Called by auto-recovery, restart-after-TTS, and visibility-resume paths.
const startCurrentSTT = useCallback((lang: string) => {
if (sttProvider === 'server') {
void audioStreamService.startListening(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else if (sttProvider === 'wasm') {
void wasmSttService.startListening(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else if (isMobile) {
browserVoiceService.startListeningSync(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
browserVoiceService.startListening(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
}
}, [sttProvider, isMobile]);
// Handle speech recognition error
const handleSpeechError = useCallback((errorMsg: string) => {
// Ignore errors if we've already stopped voice mode
@@ -316,26 +337,59 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
setStatus('idle');
return;
}
// Network / server-unreachable errors: don't retry at all.
// The user must fix connectivity and then manually restart voice.
const isNetworkError =
normalizedError.includes('network') ||
normalizedError.includes('connection') ||
normalizedError.includes('check connection');
if (isNetworkError) {
console.error('[useBrowserVoice] Network error — staying in error state:', errorMsg);
setError(errorMsg);
setStatus('error');
consecutiveRecoveryRetriesRef.current = 0;
if (recoveryTimerRef.current !== null) {
clearTimeout(recoveryTimerRef.current);
recoveryTimerRef.current = null;
}
return;
}
console.error('[useBrowserVoice] Recognition error:', errorMsg);
setError(errorMsg);
setStatus('error');
// Auto-recover from certain errors
if (!errorMsg.includes('permission') && !errorMsg.includes('not allowed')) {
setTimeout(() => {
if (isActiveRef.current) {
setStatus('listening');
setError(null);
if (sttProvider === 'server') {
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechError).catch(() => {});
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
}
}
}, 1000);
// Auto-recover from transient / non-permission errors with limited retries.
// Skip recovery when finalising — the user explicitly stopped voice.
if (isPermissionStyleError || isFinalizingRef.current) {
consecutiveRecoveryRetriesRef.current = 0;
return;
}
}, [language, conversationMode, sttProvider]);
const nextRetry = consecutiveRecoveryRetriesRef.current + 1;
consecutiveRecoveryRetriesRef.current = nextRetry;
const MAX_RECOVERY_RETRIES = 3;
if (nextRetry <= MAX_RECOVERY_RETRIES) {
const delay = Math.min(1000 * Math.pow(2, nextRetry - 1), 8000);
console.log(`[useBrowserVoice] Scheduling recovery retry ${nextRetry}/${MAX_RECOVERY_RETRIES} in ${delay}ms`);
if (recoveryTimerRef.current !== null) {
clearTimeout(recoveryTimerRef.current);
}
recoveryTimerRef.current = setTimeout(() => {
recoveryTimerRef.current = null;
if (!isActiveRef.current) return;
setStatus('listening');
setError(null);
startCurrentSTT(language);
}, delay);
} else {
console.log('[useBrowserVoice] Max recovery retries reached — staying in error state');
}
}, [language, conversationMode, startCurrentSTT]);
// Update the ref when handleSpeechError changes
useEffect(() => {
@@ -362,7 +416,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
// Non-continuous mode: fill chat input only, do not auto-send.
if (!conversationMode) {
setPendingInputText(finalText.trim(), 'replace');
setPendingInputText(finalText.trim(), 'append-inline');
processingMessageRef.current = false;
isActiveRef.current = false;
setStatus('idle');
@@ -446,19 +500,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
setStatus('listening');
if (sttProvider === 'server') {
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
console.error('[useBrowserVoice] Failed to restart server STT:', err);
});
} else if (isMobile) {
try {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} catch (err) {
console.error('[useBrowserVoice] Failed to restart listening:', err);
}
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
}
startCurrentSTT(language);
} else {
// In non-continuous mode, return to idle after AI responds
isActiveRef.current = false;
@@ -547,19 +589,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
// Only restart listening if conversation mode is enabled
if (conversationMode) {
setStatus('listening');
if (sttProvider === 'server') {
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
console.error('[useBrowserVoice] Failed to restart server STT after speech error:', err);
});
} else if (isMobile) {
try {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} catch (restartErr) {
console.error('[useBrowserVoice] Failed to restart listening after speech error:', restartErr);
}
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
}
startCurrentSTT(language);
} else {
// In non-continuous mode, return to idle after error
isActiveRef.current = false;
@@ -583,7 +613,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
setStatus('error');
processingMessageRef.current = false;
}
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, openaiCompatibleTtsModel, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, sttProvider]);
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, openaiCompatibleTtsModel, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, startCurrentSTT]);
// Handle speech recognition result
const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
@@ -591,6 +621,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
const normalized = text.trim();
if (!isFinal || !normalized) return;
console.log('[useBrowserVoice] Speech result:', normalized);
pendingFinalTranscriptRef.current = normalized;
if (finalTranscriptTimerRef.current) {
@@ -626,13 +657,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
pendingResumeOnVisibleRef.current = false;
setStatus('listening');
try {
if (sttProvider === 'server') {
void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else if (isMobile) {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
}
startCurrentSTT(language);
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Failed to resume voice';
setError(errorMsg);
@@ -644,7 +669,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return () => {
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, [conversationMode, isMobile, language, sttProvider]);
}, [conversationMode, isMobile, language, sttProvider, startCurrentSTT]);
useEffect(() => {
if (typeof navigator === 'undefined') {
@@ -679,17 +704,10 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
try {
if (sttProvider === 'server') {
audioStreamService.stopListening();
void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
browserVoiceService.stopListening();
if (isMobile) {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
}
}
browserVoiceService.stopListening();
audioStreamService.stopListening();
wasmSttService.stopListening();
startCurrentSTT(language);
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
setError(errorMsg);
@@ -708,7 +726,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
deviceChangeRestartTimerRef.current = null;
}
};
}, [isMobile, language, status, sttProvider]);
}, [isMobile, language, status, sttProvider, startCurrentSTT]);
// Update the ref when handleSpeechResult changes
useEffect(() => {
@@ -720,8 +738,8 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
if (!isSupported) {
return false;
}
if (sttProvider === 'server') {
// getUserMedia permission is requested on startListening; nothing to prepare
if (sttProvider === 'server' || sttProvider === 'wasm') {
// Permission is requested on startListening; nothing to pre-prepare
return true;
}
try {
@@ -750,6 +768,12 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
isActiveRef.current = true;
lastTranscriptRef.current = '';
consecutiveRecoveryRetriesRef.current = 0;
isFinalizingRef.current = false;
if (recoveryTimerRef.current !== null) {
clearTimeout(recoveryTimerRef.current);
recoveryTimerRef.current = null;
}
setError(null);
setStatus('listening');
@@ -774,6 +798,40 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return;
}
if (sttProvider === 'wasm') {
// WASM STT: ensure model is loaded then start recording
const modelStatus = wasmSttService.getModelStatus();
console.log('[useBrowserVoice] WASM model status:', modelStatus.state);
if (modelStatus.state !== 'ready') {
try {
setStatus('processing');
await wasmSttService.loadModel(wasmSttModel);
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Failed to load Whisper model';
console.error('[useBrowserVoice] WASM model load error:', errorMsg);
setError(errorMsg);
setStatus('error');
isActiveRef.current = false;
return;
}
}
wasmSttService.configure({
silenceThresholdDb: sttSilenceThresholdDb,
silenceHoldMs: sttSilenceHoldMs,
});
try {
await wasmSttService.startListening(language, handleSpeechResult, handleSpeechError);
console.log('[useBrowserVoice] WASM listening started');
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
console.error('[useBrowserVoice] WASM STT start error:', errorMsg);
setError(errorMsg);
setStatus('error');
isActiveRef.current = false;
}
return;
}
// Browser STT
// On mobile, use sync path to ensure SpeechRecognition.start() is called
// within the same user gesture context (required by iOS Safari)
@@ -812,17 +870,23 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
isActiveRef.current = false;
}
}
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]);
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, wasmSttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]);
// Stop voice mode
const stopVoice = useCallback(() => {
isActiveRef.current = false;
processingMessageRef.current = false;
pendingResumeOnVisibleRef.current = false;
consecutiveRecoveryRetriesRef.current = 0;
isFinalizingRef.current = false;
if (deviceChangeRestartTimerRef.current) {
clearTimeout(deviceChangeRestartTimerRef.current);
deviceChangeRestartTimerRef.current = null;
}
if (recoveryTimerRef.current !== null) {
clearTimeout(recoveryTimerRef.current);
recoveryTimerRef.current = null;
}
pendingFinalTranscriptRef.current = '';
if (finalTranscriptTimerRef.current) {
clearTimeout(finalTranscriptTimerRef.current);
@@ -830,6 +894,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
browserVoiceService.stopListening();
audioStreamService.stopListening();
wasmSttService.stopListening();
browserVoiceService.cancelSpeech();
stopServerTTS(); // Also stop server TTS if playing
stopSayTTS(); // Also stop Say TTS if playing
@@ -842,6 +907,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return;
}
isFinalizingRef.current = true;
pendingResumeOnVisibleRef.current = false;
if (deviceChangeRestartTimerRef.current) {
clearTimeout(deviceChangeRestartTimerRef.current);
@@ -869,6 +935,28 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return;
}
if (sttProvider === 'wasm') {
// Inference runs in a Web Worker — no main-thread freeze.
void wasmSttService.finishListening().then(() => {
window.setTimeout(() => {
if (!isActiveRef.current) {
return;
}
if (pendingFinalTranscriptRef.current || finalTranscriptTimerRef.current) {
return;
}
if (processingMessageRef.current) {
return;
}
isActiveRef.current = false;
processingMessageRef.current = false;
setStatus('idle');
isFinalizingRef.current = false;
}, FINAL_TRANSCRIPT_SETTLE_MS + 200);
});
return;
}
browserVoiceService.stopListening();
window.setTimeout(() => {
if (!isActiveRef.current) {
+2 -1
View File
@@ -166,9 +166,10 @@ export type DesktopSettings = {
responseStyleEnabled?: boolean;
responseStylePreset?: 'concise' | 'detailed' | 'mentor' | 'pushback' | 'noFiller' | 'matchEnergy' | 'warmPeer' | 'custom';
responseStyleCustomInstructions?: string;
sttProvider?: 'browser' | 'server';
sttProvider?: 'browser' | 'server' | 'wasm';
sttServerUrl?: string;
sttModel?: string;
wasmSttModel?: string;
sttLanguage?: string;
sttSilenceThresholdDb?: number;
sttSilenceHoldMs?: number;
@@ -1301,6 +1301,14 @@ export const settingsDict = {
'settings.voice.page.provider.custom': 'Custom',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': 'Server',
'settings.voice.page.provider.wasm': 'Local',
'settings.voice.page.stt.wasmModel': 'Whisper Model',
'settings.voice.page.stt.wasmLoaded': 'Model loaded, ready',
'settings.voice.page.stt.wasmDownloading': 'Downloading model...',
'settings.voice.page.stt.wasmLoading': 'Loading model...',
'settings.voice.page.stt.wasmNotLoaded': 'Model will load on first voice use',
'settings.voice.page.stt.wasmDownload': 'Load',
'settings.voice.page.stt.wasmRetry': 'Retry',
'settings.voice.page.tooltip.browser': 'Free, offline, limited mobile support.',
'settings.voice.page.tooltip.openai': 'High quality, mobile ready, needs API key.',
'settings.voice.page.tooltip.custom': 'OpenAI-compatible server (for example Kokoro).',
@@ -1301,6 +1301,14 @@ export const settingsDict = {
"settings.voice.page.provider.custom": "Personalizado",
"settings.voice.page.provider.say": "Decir",
"settings.voice.page.provider.server": "Servidor",
"settings.voice.page.provider.wasm": "Local",
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
"settings.voice.page.stt.wasmLoaded": "Modelo cargado",
"settings.voice.page.stt.wasmDownloading": "Descargando modelo...",
"settings.voice.page.stt.wasmLoading": "Cargando modelo...",
"settings.voice.page.stt.wasmNotLoaded": "El modelo se descargará en el primer uso de voz",
"settings.voice.page.stt.wasmDownload": "Descargar",
"settings.voice.page.stt.wasmRetry": "Reintentar",
"settings.voice.page.tooltip.browser": "Gratuito, offline, soporte limitado en móvil.",
"settings.voice.page.tooltip.openai": "Calidad alta, compatible con móviles, requiere clave de API.",
"settings.voice.page.tooltip.custom": "Servidor compatible con OpenAI (por ejemplo Kokoro).",
@@ -1301,6 +1301,14 @@ export const settingsDict = {
'settings.voice.page.provider.custom': '사용자 정의',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': '서버',
'settings.voice.page.provider.wasm': '로컬',
'settings.voice.page.stt.wasmModel': 'Whisper 모델',
'settings.voice.page.stt.wasmLoaded': '모델 로드됨',
'settings.voice.page.stt.wasmDownloading': '모델 다운로드 중...',
'settings.voice.page.stt.wasmLoading': '모델 로드 중...',
'settings.voice.page.stt.wasmNotLoaded': '첫 음성 사용 시 모델이 다운로드됩니다',
'settings.voice.page.stt.wasmDownload': '다운로드',
'settings.voice.page.stt.wasmRetry': '재시도',
'settings.voice.page.tooltip.browser': '무료, 오프라인, 모바일 지원 제한.',
'settings.voice.page.tooltip.openai': '고품질, 모바일 지원, API key 필요.',
'settings.voice.page.tooltip.custom': 'OpenAI 호환 서버(예: Kokoro).',
@@ -1527,6 +1527,14 @@ export const settingsDict = {
'settings.voice.page.provider.custom': 'Własny',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': 'Serwer',
'settings.voice.page.provider.wasm': 'Lokalny',
'settings.voice.page.stt.wasmModel': 'Model Whisper',
'settings.voice.page.stt.wasmLoaded': 'Model załadowany',
'settings.voice.page.stt.wasmDownloading': 'Pobieranie modelu...',
'settings.voice.page.stt.wasmLoading': 'Ładowanie modelu...',
'settings.voice.page.stt.wasmNotLoaded': 'Model zostanie pobrany przy pierwszym użyciu',
'settings.voice.page.stt.wasmDownload': 'Pobierz',
'settings.voice.page.stt.wasmRetry': 'Spróbuj ponownie',
'settings.voice.page.section.playbackAndSummary': 'Odtwarzanie i podsumowanie',
'settings.voice.page.section.speechRecognition': 'Rozpoznawanie mowy',
'settings.voice.page.section.voiceSetup': 'Konfiguracja głosu',
@@ -1301,6 +1301,14 @@ export const settingsDict = {
"settings.voice.page.provider.custom": "Personalizado",
"settings.voice.page.provider.say": "Falar",
"settings.voice.page.provider.server": "Servidor",
"settings.voice.page.provider.wasm": "Local",
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
"settings.voice.page.stt.wasmLoaded": "Modelo carregado",
"settings.voice.page.stt.wasmDownloading": "Baixando modelo...",
"settings.voice.page.stt.wasmLoading": "Carregando modelo...",
"settings.voice.page.stt.wasmNotLoaded": "O modelo será baixado no primeiro uso",
"settings.voice.page.stt.wasmDownload": "Baixar",
"settings.voice.page.stt.wasmRetry": "Tentar novamente",
"settings.voice.page.tooltip.browser": "Gratuito, offline, suporte limitado em dispositivos móveis.",
"settings.voice.page.tooltip.openai": "Alta qualidade, compatível com dispositivos móveis, exige chave de API.",
"settings.voice.page.tooltip.custom": "Servidor compatível com OpenAI (por exemplo Kokoro).",
@@ -1301,6 +1301,14 @@ export const settingsDict = {
"settings.voice.page.provider.custom": "Власний",
"settings.voice.page.provider.say": "Say",
"settings.voice.page.provider.server": "Сервер",
"settings.voice.page.provider.wasm": "Локально",
"settings.voice.page.stt.wasmModel": "Модель Whisper",
"settings.voice.page.stt.wasmLoaded": "Модель завантажено",
"settings.voice.page.stt.wasmDownloading": "Завантаження моделі...",
"settings.voice.page.stt.wasmLoading": "Завантаження моделі...",
"settings.voice.page.stt.wasmNotLoaded": "Модель завантажиться при першому використанні",
"settings.voice.page.stt.wasmDownload": "Завантажити",
"settings.voice.page.stt.wasmRetry": "Спробувати знову",
"settings.voice.page.tooltip.browser": "Безкоштовна, офлайн, обмежена мобільна підтримка.",
"settings.voice.page.tooltip.openai": "Висока якість, готово для мобільних пристроїв, потрібен API ключ.",
"settings.voice.page.tooltip.custom": "OpenAI-сумісний сервер, наприклад Kokoro.",
@@ -1301,6 +1301,14 @@ export const settingsDict = {
'settings.voice.page.provider.custom': '自定义',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': '服务器',
'settings.voice.page.provider.wasm': '本地',
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
'settings.voice.page.stt.wasmLoaded': '模型已加载',
'settings.voice.page.stt.wasmDownloading': '下载模型中...',
'settings.voice.page.stt.wasmLoading': '加载模型中...',
'settings.voice.page.stt.wasmNotLoaded': '模型将在首次使用时下载',
'settings.voice.page.stt.wasmDownload': '下载',
'settings.voice.page.stt.wasmRetry': '重试',
'settings.voice.page.tooltip.browser': '免费、离线,移动端支持有限。',
'settings.voice.page.tooltip.openai': '质量高,移动端可用,需要 API Key。',
'settings.voice.page.tooltip.custom': 'OpenAI 兼容服务器(例如 Kokoro)。',
@@ -282,8 +282,10 @@ class BrowserVoiceService {
const errorMessage = this.getErrorMessage(event.error);
this.onErrorCallback?.(errorMessage);
// Don't restart on certain errors
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') {
// Don't restart on fatal / unrecoverable errors.
// "network" in Electron/Chromium means Google's speech servers are unreachable;
// auto-restarting immediately creates an infinite error → end → start → error loop.
if (event.error === 'not-allowed' || event.error === 'service-not-allowed' || event.error === 'network') {
this.restartOnEnd = false;
this.isListening = false;
}
@@ -292,12 +294,13 @@ class BrowserVoiceService {
this.recognition.onend = () => {
this.isListening = false;
// Auto-restart if still supposed to be listening and not speaking
// Auto-restart if still supposed to be listening and not speaking.
// Only restart when we have a valid recognition instance and restartOnEnd is set.
if (this.restartOnEnd && this.recognition && !this.isSpeaking) {
try {
this.recognition.start();
} catch {
// Ignore restart errors
// Ignore restart errors — onerror / onend will fire if it's fatal.
}
}
};
+556
View File
@@ -0,0 +1,556 @@
/**
* WASM Speech-to-Text Service
*
* Local Whisper transcription via Transformers.js (ONNX Runtime Web).
* Captures microphone audio, detects utterance boundaries via silence-based
* VAD, then transcribes each utterance locally no cloud API required.
*
* Works in Electron and all modern browsers that support Web Audio API.
* First use downloads a Whisper model (~40166 MB, cached).
*/
export type WasmModelStatus =
| { state: 'unloaded' }
| { state: 'downloading'; progress: number }
| { state: 'loading' }
| { state: 'ready' }
| { state: 'error'; error: string };
export interface WasmModelInfo {
id: string;
name: string;
size: string;
languages: string;
description: string;
}
export const WASM_MODELS: WasmModelInfo[] = [
{
id: 'Xenova/whisper-tiny.en',
name: 'Whisper Tiny (EN)',
size: '~39 MB',
languages: 'English',
description: 'Fastest, lowest accuracy. Good for quick dictation.',
},
{
id: 'Xenova/whisper-base.en',
name: 'Whisper Base (EN)',
size: '~73 MB',
languages: 'English',
description: 'Balanced speed and accuracy. Default for English.',
},
{
id: 'Xenova/whisper-small.en',
name: 'Whisper Small (EN)',
size: '~166 MB',
languages: 'English',
description: 'Higher accuracy, slower. Best for noisy environments.',
},
];
export type SpeechResultCallback = (text: string, isFinal: boolean) => void;
export type ErrorCallback = (error: string) => void;
const VAD_POLL_MS = 80;
const MIN_UTTERANCE_MS = 300;
const WHISPER_SAMPLE_RATE = 16000;
interface WasmSttConfig {
silenceThresholdDb?: number;
silenceHoldMs?: number;
}
class WasmSttService {
private transcriber: unknown = null;
private worker: Worker | null = null;
private modelStatus: WasmModelStatus = { state: 'unloaded' };
private currentModelId: string | null = null;
private stream: MediaStream | null = null;
private mediaRecorder: MediaRecorder | null = null;
private audioContext: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private vadTimer: ReturnType<typeof setInterval> | null = null;
private chunks: Blob[] = [];
private recordingStartMs = 0;
private isActive = false;
private isSpeaking = false;
private silenceSince: number | null = null;
private onResult: SpeechResultCallback | null = null;
private onError: ErrorCallback | null = null;
private finishResolver: (() => void) | null = null;
private lang = 'en';
private cfg: Required<WasmSttConfig> = {
silenceThresholdDb: -45,
silenceHoldMs: 1500,
};
public onModelStatusChange: ((status: WasmModelStatus) => void) | null = null;
configure(config: WasmSttConfig): void {
this.cfg = { ...this.cfg, ...config };
}
isSupported(): boolean {
return (
typeof window !== 'undefined' &&
typeof navigator !== 'undefined' &&
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
typeof window.MediaRecorder !== 'undefined' &&
typeof window.AudioContext !== 'undefined'
);
}
getModelStatus(): WasmModelStatus {
return this.modelStatus;
}
getCurrentModelId(): string | null {
return this.currentModelId;
}
private setModelStatus(status: WasmModelStatus): void {
this.modelStatus = status;
this.onModelStatusChange?.(status);
}
async loadModel(modelId: string): Promise<void> {
if (this.currentModelId === modelId && this.modelStatus.state === 'ready') {
return;
}
if (this.modelStatus.state === 'downloading' || this.modelStatus.state === 'loading') {
return;
}
this._terminateWorker();
this.transcriber = null;
this.setModelStatus({ state: 'downloading', progress: 0 });
this.currentModelId = modelId;
// Try Web Worker first — inference off main thread = no UI freeze.
try {
const WasmWorkerMod = await import('./wasmSttWorker?worker');
const WasmWorker = WasmWorkerMod.default as new () => Worker;
this.worker = new WasmWorker();
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('Worker init timed out')), 10000);
this.worker!.onmessage = (e: MessageEvent) => {
const data = e.data as { type: string; progress?: number; error?: string; text?: string };
if (data.type === 'progress') {
this.setModelStatus({ state: 'downloading', progress: data.progress ?? 0 });
} else if (data.type === 'loaded') {
clearTimeout(timer);
resolve();
} else if (data.type === 'error') {
clearTimeout(timer);
reject(new Error(data.error ?? 'Worker load failed'));
}
};
this.worker!.onerror = (err) => {
clearTimeout(timer);
reject(new Error(err.message || 'Worker error'));
};
this.worker!.postMessage({ type: 'load', modelId });
});
this.setModelStatus({ state: 'ready' });
return;
} catch (err) {
console.warn('[WasmStt] Worker failed, using main-thread:', err instanceof Error ? err.message : err);
this._terminateWorker();
}
// Fallback: main-thread pipeline (causes brief UI freeze during inference).
try {
const { pipeline, env } = await import('@xenova/transformers');
env.backends.onnx.wasm.numThreads = 1;
env.allowLocalModels = false;
const fileDoneBytes = new Map<string, number>();
let totalDone = 0;
let totalEstimate = 0;
this.transcriber = await pipeline('automatic-speech-recognition', modelId, {
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
if (info.status === 'progress' && info.file) {
const prevDone = fileDoneBytes.get(info.file) ?? 0;
const currentDone = info.loaded ?? 0;
const delta = Math.max(0, currentDone - prevDone);
fileDoneBytes.set(info.file, currentDone);
totalDone += delta;
if (info.total && info.total > totalEstimate) totalEstimate = info.total;
const effectiveTotal = Math.max(totalEstimate, totalDone);
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
this.setModelStatus({ state: 'downloading', progress: pct });
}
},
});
this.setModelStatus({ state: 'ready' });
} catch (err) {
const msg = err instanceof Error ? err.message : 'Unknown error loading model';
this.setModelStatus({ state: 'error', error: msg });
this.transcriber = null;
this.currentModelId = null;
throw err;
}
}
private _terminateWorker(): void {
if (this.worker) {
this.worker.terminate();
this.worker = null;
}
}
async unloadModel(): Promise<void> {
this._terminateWorker();
this.transcriber = null;
this.currentModelId = null;
this.setModelStatus({ state: 'unloaded' });
}
async startListening(
lang: string,
onResult: SpeechResultCallback,
onError?: ErrorCallback,
): Promise<void> {
if (this.isActive) {
this.stopListening();
}
if (!this.transcriber && !this.worker) {
onError?.('Whisper model not loaded. Select a model in Voice Settings first.');
return;
}
this.lang = lang;
this.onResult = onResult;
this.onError = onError ?? null;
this.isActive = true;
try {
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
} catch (err) {
this.isActive = false;
const msg = err instanceof Error ? err.message : 'Microphone access denied';
onError?.(msg);
return;
}
this._setupAudioContext();
this._startRecorder();
this._startVAD();
}
stopListening(): void {
this._stopVAD();
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
try { this.mediaRecorder.stop(); } catch { /* ignore */ }
}
this._cleanupAfterStop(true);
}
async finishListening(): Promise<void> {
if (!this.isActive) return;
this._stopVAD();
this.isSpeaking = false;
this.silenceSince = null;
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
this._cleanupAfterStop(true);
return;
}
await new Promise<void>((resolve) => {
this.finishResolver = resolve;
this._finaliseUtterance(false);
});
this._cleanupAfterStop(true);
}
getIsListening(): boolean {
return this.isActive;
}
// ── Audio capture ────────────────────────────────────────────────────
private _setupAudioContext(): void {
if (!this.stream) return;
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
this.audioContext = new AudioContextClass();
const source = this.audioContext.createMediaStreamSource(this.stream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 512;
source.connect(this.analyser);
}
private _teardownAudioContext(): void {
try { this.audioContext?.close(); } catch { /* ignore */ }
this.audioContext = null;
this.analyser = null;
}
private _startRecorder(): void {
if (!this.stream) return;
const mimeType = this._pickMimeType();
const options: MediaRecorderOptions = {};
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
options.mimeType = mimeType;
}
this.mediaRecorder = new MediaRecorder(this.stream, options);
this.chunks = [];
this.recordingStartMs = Date.now();
this.mediaRecorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) {
this.chunks.push(e.data);
}
};
this.mediaRecorder.onstop = () => {
const blobs = this.chunks.splice(0);
const durationMs = Date.now() - this.recordingStartMs;
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
this.finishResolver?.();
this.finishResolver = null;
return;
}
const mType = blobs[0].type || mimeType || 'audio/webm';
const blob = new Blob(blobs, { type: mType });
void this._transcribe(blob).finally(() => {
this.finishResolver?.();
this.finishResolver = null;
});
};
this.mediaRecorder.start(250);
}
private _releaseStream(): void {
if (this.stream) {
this.stream.getTracks().forEach((t) => t.stop());
this.stream = null;
}
}
// ── VAD ──────────────────────────────────────────────────────────────
private _startVAD(): void {
this._stopVAD();
this.silenceSince = null;
this.isSpeaking = false;
this.vadTimer = setInterval(() => {
if (!this.isActive || !this.analyser) return;
const db = this._getRmsDb();
const isSilent = db < this.cfg.silenceThresholdDb;
if (!isSilent) {
this.silenceSince = null;
if (!this.isSpeaking) {
this.isSpeaking = true;
if (this.mediaRecorder?.state === 'recording') {
this.recordingStartMs = Date.now();
}
}
} else {
if (this.isSpeaking) {
if (this.silenceSince === null) {
this.silenceSince = Date.now();
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
this.isSpeaking = false;
this.silenceSince = null;
this._finaliseUtterance(true);
}
}
}
}, VAD_POLL_MS);
}
private _stopVAD(): void {
if (this.vadTimer !== null) {
clearInterval(this.vadTimer);
this.vadTimer = null;
}
}
private _cleanupAfterStop(clearChunks: boolean): void {
const pendingResolver = this.finishResolver;
this.isActive = false;
this.finishResolver = null;
this.mediaRecorder = null;
this._teardownAudioContext();
this._releaseStream();
if (clearChunks) this.chunks = [];
this.isSpeaking = false;
this.silenceSince = null;
this.onResult = null;
this.onError = null;
pendingResolver?.();
}
private _finaliseUtterance(restart: boolean): void {
if (!this.isActive) return;
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
this.mediaRecorder.stop();
}
if (!restart) return;
setTimeout(() => {
if (this.isActive && this.stream) {
this._startRecorder();
}
}, 100);
}
private _getRmsDb(): number {
if (!this.analyser) return -Infinity;
const buf = new Float32Array(this.analyser.fftSize);
this.analyser.getFloatTimeDomainData(buf);
let sumSq = 0;
for (const s of buf) sumSq += s * s;
const rms = Math.sqrt(sumSq / buf.length);
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
}
// ── Transcription ────────────────────────────────────────────────────
private async _transcribe(blob: Blob): Promise<void> {
if (!this.onResult) return;
if (!this.transcriber && !this.worker) {
this.onError?.('Model not loaded');
return;
}
try {
const audioData = await this._decodeToFloat32(blob);
if (!audioData || audioData.length === 0) {
this.onError?.(`Failed to decode audio (${blob.size} bytes)`);
return;
}
const langHint = this._resolveLanguageHint();
// Prefer worker (non-blocking); fall back to main-thread pipeline.
const transcript = this.worker
? await this._transcribeViaWorker(audioData, langHint)
: await this._transcribeMainThread(audioData, langHint);
if (transcript) {
this.onResult(transcript, true);
}
} catch (err) {
if (!this.isActive) return;
const msg = err instanceof Error ? err.message : 'Local transcription failed';
this.onError?.(msg);
}
}
private _transcribeViaWorker(audioData: Float32Array, langHint: string | undefined): Promise<string> {
return new Promise((resolve, reject) => {
if (!this.worker) return reject(new Error('Worker gone'));
const onMessage = (e: MessageEvent) => {
const data = e.data as { type: string; error?: string; transcript?: string; text?: string };
if (data.type === 'result') {
this.worker!.removeEventListener('message', onMessage);
resolve(data.transcript ?? '');
} else if (data.type === 'log') {
console.log('[WasmStt Worker]', data.text);
} else if (data.type === 'error') {
this.worker!.removeEventListener('message', onMessage);
reject(new Error(data.error ?? 'Transcription failed'));
}
};
this.worker.addEventListener('message', onMessage);
this.worker.postMessage(
{ type: 'transcribe', audio: audioData.buffer, language: langHint },
[audioData.buffer],
);
setTimeout(() => {
this.worker?.removeEventListener('message', onMessage);
reject(new Error('Transcription timed out'));
}, 30000);
});
}
private async _transcribeMainThread(audioData: Float32Array, langHint: string | undefined): Promise<string> {
const pipelineFn = this.transcriber as (
input: Float32Array,
options?: Record<string, unknown>,
) => Promise<{ text: string }>;
const result = await pipelineFn(audioData, {
task: 'transcribe',
...(langHint ? { language: langHint } : {}),
});
return (result?.text ?? '').trim();
}
private async _decodeToFloat32(blob: Blob): Promise<Float32Array | null> {
if (!this.audioContext) return null;
const arrayBuffer = await blob.arrayBuffer();
let audioBuffer: AudioBuffer;
try {
audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
} catch {
return null;
}
const origRate = audioBuffer.sampleRate;
const origData = audioBuffer.getChannelData(0);
const targetRate = WHISPER_SAMPLE_RATE;
if (origRate === targetRate) {
return new Float32Array(origData);
}
const ratio = origRate / targetRate;
const newLength = Math.ceil(origData.length / ratio);
const result = new Float32Array(newLength);
for (let i = 0; i < newLength; i++) {
const origIdx = i * ratio;
const idx0 = Math.floor(origIdx);
const idx1 = Math.min(idx0 + 1, origData.length - 1);
const frac = origIdx - idx0;
result[i] = origData[idx0] * (1 - frac) + origData[idx1] * frac;
}
return result;
}
private _resolveLanguageHint(): string | undefined {
if (this.lang && this.lang !== 'auto') {
return this.lang.split('-')[0];
}
return undefined;
}
private _pickMimeType(): string {
const candidates = [
'audio/webm;codecs=opus',
'audio/webm',
'audio/ogg;codecs=opus',
'audio/ogg',
'audio/mp4',
];
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
}
return '';
}
}
export const wasmSttService = new WasmSttService();
export { WasmSttService };
@@ -0,0 +1,92 @@
/**
* Web Worker for off-main-thread Whisper transcription.
*
* Receives `{ type: 'load', modelId }` to load a model, then
* `{ type: 'transcribe', audio: Float32Array (transferred buffer), language? }`
* to run inference. Posts progress, results, and errors back.
*/
import { pipeline, env } from '@xenova/transformers';
let transcriber: unknown = null;
self.onmessage = async (e: MessageEvent) => {
const { type } = e.data as { type: string };
if (type === 'load') {
const { modelId } = e.data as { modelId: string };
try {
env.backends.onnx.wasm.numThreads = 1;
const fileDoneBytes = new Map<string, number>();
let totalDone = 0;
let totalEstimate = 0;
transcriber = await pipeline('automatic-speech-recognition', modelId, {
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
if (info.status === 'progress' && info.file) {
const prevDone = fileDoneBytes.get(info.file) ?? 0;
const currentDone = info.loaded ?? 0;
const delta = Math.max(0, currentDone - prevDone);
fileDoneBytes.set(info.file, currentDone);
totalDone += delta;
if (info.total && info.total > totalEstimate) {
totalEstimate = info.total;
}
const effectiveTotal = Math.max(totalEstimate, totalDone);
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
self.postMessage({ type: 'progress', progress: pct });
}
},
});
self.postMessage({ type: 'loaded' });
} catch (err) {
self.postMessage({
type: 'error',
error: err instanceof Error ? err.message : 'Failed to load model',
});
}
} else if (type === 'transcribe') {
if (!transcriber) {
self.postMessage({ type: 'error', error: 'Model not loaded', seq: (e.data as { seq?: number }).seq });
return;
}
const { audio, language, seq } = e.data as { audio: ArrayBuffer; language?: string; seq?: number };
try {
const samples = new Float32Array(audio);
if (samples.length === 0) {
self.postMessage({ type: 'error', error: 'Empty audio received', seq });
return;
}
self.postMessage({ type: 'log', text: `Transcribing ${samples.length} samples (${(samples.length / 16000).toFixed(1)}s)` });
const pipelineFn = transcriber as (
input: Float32Array,
options?: Record<string, unknown>,
) => Promise<{ text: string }>;
const result = await pipelineFn(samples, {
task: 'transcribe',
...(language ? { language } : {}),
});
self.postMessage({
type: 'result',
transcript: (result?.text ?? '').trim(),
seq,
});
} catch (err) {
self.postMessage({
type: 'error',
error: err instanceof Error ? err.message : 'Transcription failed',
seq,
});
}
}
};
+29 -7
View File
@@ -50,8 +50,9 @@ interface OpenChamberDefaults {
defaultFileViewerPreview?: boolean;
zenModel?: string;
messageStreamTransport?: 'auto' | 'ws' | 'sse';
sttProvider?: 'browser' | 'server';
sttProvider?: 'browser' | 'server' | 'wasm';
sttServerUrl?: string;
wasmSttModel?: string;
sttModel?: string;
sttLanguage?: string;
sttSilenceThresholdDb?: number;
@@ -77,7 +78,7 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
? data.messageStreamTransport
: undefined;
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' ? data.sttProvider : undefined;
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' || data?.sttProvider === 'wasm' ? data.sttProvider : undefined;
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
@@ -552,9 +553,10 @@ interface ConfigStore {
openaiCompatibleVoice: string;
openaiCompatibleTtsModel: string;
// STT (speech-to-text) settings
sttProvider: 'browser' | 'server';
sttProvider: 'browser' | 'server' | 'wasm';
sttServerUrl: string;
sttModel: string;
wasmSttModel: string;
sttLanguage: string;
sttSilenceThresholdDb: number;
sttSilenceHoldMs: number;
@@ -576,9 +578,10 @@ interface ConfigStore {
setOpenaiCompatibleUrl: (url: string) => void;
setOpenaiCompatibleVoice: (voice: string) => void;
setOpenaiCompatibleTtsModel: (model: string) => void;
setSttProvider: (provider: 'browser' | 'server') => void;
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => void;
setSttServerUrl: (url: string) => void;
setSttModel: (model: string) => void;
setWasmSttModel: (model: string) => void;
setSttLanguage: (lang: string) => void;
setSttSilenceThresholdDb: (db: number) => void;
setSttSilenceHoldMs: (ms: number) => void;
@@ -761,11 +764,15 @@ export const useConfigStore = create<ConfigStore>()(
}
return 'kokoro';
})(),
// STT provider: 'browser' (Web Speech API) or 'server' (OpenAI-compat)
// STT provider: 'browser' (Web Speech API), 'server' (OpenAI-compat), 'wasm' (local Whisper)
sttProvider: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('sttProvider');
if (saved === 'browser' || saved === 'server') return saved;
if (saved === 'browser' || saved === 'server' || saved === 'wasm') return saved;
// Electron/Chromium's Web Speech API requires Google API keys
// not available in Electron, so default to WASM local Whisper.
const electron = (window as unknown as { __OPENCHAMBER_ELECTRON__?: { runtime?: string } }).__OPENCHAMBER_ELECTRON__;
if (electron?.runtime === 'electron') return 'wasm' as const;
}
return 'browser' as const;
})(),
@@ -783,6 +790,13 @@ export const useConfigStore = create<ConfigStore>()(
}
return 'deepdml/faster-whisper-large-v3-turbo-ct2';
})(),
wasmSttModel: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('wasmSttModel');
if (saved) return saved;
}
return 'Xenova/whisper-base.en';
})(),
sttLanguage: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('sttLanguage');
@@ -1886,7 +1900,7 @@ export const useConfigStore = create<ConfigStore>()(
}
},
setSttProvider: (provider: 'browser' | 'server') => {
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => {
set({ sttProvider: provider });
if (typeof window !== 'undefined') {
localStorage.setItem('sttProvider', provider);
@@ -1910,6 +1924,14 @@ export const useConfigStore = create<ConfigStore>()(
updateDesktopSettings({ sttModel: model }).catch(() => {});
},
setWasmSttModel: (model: string) => {
set({ wasmSttModel: model });
if (typeof window !== 'undefined') {
localStorage.setItem('wasmSttModel', model);
}
updateDesktopSettings({ wasmSttModel: model }).catch(() => {});
},
setSttLanguage: (lang: string) => {
set({ sttLanguage: lang });
if (typeof window !== 'undefined') {
@@ -604,7 +604,7 @@ export const createSettingsHelpers = (dependencies) => {
if (typeof candidate.sttProvider === 'string') {
const provider = candidate.sttProvider.trim();
if (provider === 'browser' || provider === 'server') {
if (provider === 'browser' || provider === 'server' || provider === 'wasm') {
result.sttProvider = provider;
}
}
@@ -620,6 +620,12 @@ export const createSettingsHelpers = (dependencies) => {
result.sttModel = trimmed;
}
}
if (typeof candidate.wasmSttModel === 'string') {
const trimmed = candidate.wasmSttModel.trim();
if (trimmed.length <= 256) {
result.wasmSttModel = trimmed;
}
}
if (typeof candidate.sttLanguage === 'string') {
const trimmed = candidate.sttLanguage.trim();
if (trimmed.length <= STT_LANGUAGE_MAX_LENGTH) {