BuzzbeeSCD/openchamber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stt): add server-side STT provider via OpenAI-compatible Whisper endpoint (#860)

Browse Source 
Co-authored-by: Alexander Busse <alex@ableph.net>
This commit is contained in:
Alexander Busse
2026-04-11 22:47:33 +03:00
committed by GitHub
co-authored by Alexander Busse
parent 75a10ea66c
commit 3746d99a85
6 changed files with 836 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ui/src/hooks/useBrowserVoice.ts
+80 -18
View File
@@ -27,6 +27,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
import { audioStreamService } from '@/lib/voice/audioStreamService';
import { useSessionUIStore } from '@/sync/session-ui-store';
import { useInputStore } from '@/sync/input-store';
import { getSyncMessages, getSyncParts } from '@/sync/sync-refs';
@@ -109,8 +110,6 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return false;
});
const isSupported = browserVoiceService.isSupported();
// Mobile detection
const isMobile = useMemo(() => {
if (typeof navigator === 'undefined') return false;
@@ -143,12 +142,26 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
const sayVoice = useConfigStore((state) => state.sayVoice);
const browserVoice = useConfigStore((state) => state.browserVoice);
const openaiVoice = useConfigStore((state) => state.openaiVoice);
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
const openaiCompatibleUrl = useConfigStore((state) => state.openaiCompatibleUrl);
const summarizeVoiceConversation = useConfigStore((state) => state.summarizeVoiceConversation);
const summarizeCharacterThreshold = useConfigStore((state) => state.summarizeCharacterThreshold);
const shouldCheckOpenAIAvailability = voiceModeEnabled && voiceProvider === 'openai';
const shouldCheckSayAvailability = voiceModeEnabled && voiceProvider === 'say';
// STT provider config
const sttProvider = useConfigStore((state) => state.sttProvider);
const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
const sttModel = useConfigStore((state) => state.sttModel);
const sttLanguage = useConfigStore((state) => state.sttLanguage);
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
const isSupported = sttProvider === 'server'
? audioStreamService.isSupported()
: browserVoiceService.isSupported();
// Server TTS for mobile (bypasses Safari audio restrictions)
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS({
enabled: shouldCheckOpenAIAvailability,
@@ -170,6 +183,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
isActiveRef.current = false;
processingMessageRef.current = false;
browserVoiceService.stopListening();
audioStreamService.stopListening();
browserVoiceService.cancelSpeech();
setStatus('idle');
setError(null);
@@ -309,11 +323,15 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
if (isActiveRef.current) {
setStatus('listening');
setError(null);
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
if (sttProvider === 'server') {
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechError).catch(() => {});
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
}
}
}, 1000);
}
}, [language, conversationMode]);
}, [language, conversationMode, sttProvider]);
// Update the ref when handleSpeechError changes
useEffect(() => {
@@ -336,6 +354,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
// Stop listening while processing
browserVoiceService.stopListening();
audioStreamService.stopListening();
// Non-continuous mode: fill chat input only, do not auto-send.
if (!conversationMode) {
@@ -423,7 +442,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
setStatus('listening');
if (isMobile) {
if (sttProvider === 'server') {
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
console.error('[useBrowserVoice] Failed to restart server STT:', err);
});
} else if (isMobile) {
try {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} catch (err) {
@@ -514,7 +537,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
// Only restart listening if conversation mode is enabled
if (conversationMode) {
setStatus('listening');
if (isMobile) {
if (sttProvider === 'server') {
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
console.error('[useBrowserVoice] Failed to restart server STT after speech error:', err);
});
} else if (isMobile) {
try {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} catch (restartErr) {
@@ -546,7 +573,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
setStatus('error');
processingMessageRef.current = false;
}
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode]);
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, sttProvider]);
// Handle speech recognition result
const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
@@ -589,7 +616,9 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
pendingResumeOnVisibleRef.current = false;
setStatus('listening');
try {
if (isMobile) {
if (sttProvider === 'server') {
void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else if (isMobile) {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
@@ -605,7 +634,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return () => {
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, [conversationMode, isMobile, language]);
}, [conversationMode, isMobile, language, sttProvider]);
useEffect(() => {
if (typeof navigator === 'undefined') {
@@ -640,11 +669,16 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
try {
browserVoiceService.stopListening();
if (isMobile) {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
if (sttProvider === 'server') {
audioStreamService.stopListening();
void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
browserVoiceService.stopListening();
if (isMobile) {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
}
}
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
@@ -664,7 +698,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
deviceChangeRestartTimerRef.current = null;
}
};
}, [isMobile, language, status]);
}, [isMobile, language, status, sttProvider]);
// Update the ref when handleSpeechResult changes
useEffect(() => {
@@ -676,6 +710,10 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
if (!isSupported) {
return false;
}
if (sttProvider === 'server') {
// getUserMedia permission is requested on startListening; nothing to prepare
return true;
}
try {
await browserVoiceService.prepareListening();
return true;
@@ -684,12 +722,12 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
setError(errorMsg);
return false;
}
}, [isSupported]);
}, [isSupported, sttProvider]);
// Start voice mode
const startVoice = useCallback(async () => {
if (!isSupported) {
setError('Browser voice not supported');
setError('Voice input not supported in this browser');
setStatus('error');
return;
}
@@ -704,7 +742,29 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
lastTranscriptRef.current = '';
setError(null);
setStatus('listening');
if (sttProvider === 'server') {
// Server STT: configure the service then start async recording
audioStreamService.configure({
baseURL: sttServerUrl,
model: sttModel,
language: sttLanguage || undefined,
silenceThresholdDb: sttSilenceThresholdDb,
silenceHoldMs: sttSilenceHoldMs,
});
try {
await audioStreamService.startListening(language, handleSpeechResult, handleSpeechError);
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
console.error('[useBrowserVoice] Server STT start error:', errorMsg);
setError(errorMsg);
setStatus('error');
isActiveRef.current = false;
}
return;
}
// Browser STT
// On mobile, use sync path to ensure SpeechRecognition.start() is called
// within the same user gesture context (required by iOS Safari)
// Also unlock audio immediately for TTS playback later
@@ -742,7 +802,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
isActiveRef.current = false;
}
}
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio]);
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]);
// Stop voice mode
const stopVoice = useCallback(() => {
@@ -759,6 +819,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
finalTranscriptTimerRef.current = null;
}
browserVoiceService.stopListening();
audioStreamService.stopListening();
browserVoiceService.cancelSpeech();
stopServerTTS(); // Also stop server TTS if playing
stopSayTTS(); // Also stop Say TTS if playing
@@ -781,6 +842,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
browserVoiceService.setConversationMode(false);
browserVoiceService.stopListening();
audioStreamService.stopListening();
browserVoiceService.cancelSpeech();
};
}, []);