diff --git a/packages/ui/src/hooks/useBrowserVoice.ts b/packages/ui/src/hooks/useBrowserVoice.ts
index 41e75e9a..ab29427d 100644
--- a/packages/ui/src/hooks/useBrowserVoice.ts
+++ b/packages/ui/src/hooks/useBrowserVoice.ts
@@ -27,6 +27,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
+import { audioStreamService } from '@/lib/voice/audioStreamService';
import { useSessionUIStore } from '@/sync/session-ui-store';
import { useInputStore } from '@/sync/input-store';
import { getSyncMessages, getSyncParts } from '@/sync/sync-refs';
@@ -109,8 +110,6 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return false;
});
- const isSupported = browserVoiceService.isSupported();
-
// Mobile detection
const isMobile = useMemo(() => {
if (typeof navigator === 'undefined') return false;
@@ -143,12 +142,26 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
const sayVoice = useConfigStore((state) => state.sayVoice);
const browserVoice = useConfigStore((state) => state.browserVoice);
const openaiVoice = useConfigStore((state) => state.openaiVoice);
+ const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
+ const openaiCompatibleUrl = useConfigStore((state) => state.openaiCompatibleUrl);
const summarizeVoiceConversation = useConfigStore((state) => state.summarizeVoiceConversation);
const summarizeCharacterThreshold = useConfigStore((state) => state.summarizeCharacterThreshold);
const shouldCheckOpenAIAvailability = voiceModeEnabled && voiceProvider === 'openai';
const shouldCheckSayAvailability = voiceModeEnabled && voiceProvider === 'say';
+ // STT provider config
+ const sttProvider = useConfigStore((state) => state.sttProvider);
+ const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
+ const sttModel = useConfigStore((state) => state.sttModel);
+ const sttLanguage = useConfigStore((state) => state.sttLanguage);
+ const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
+ const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
+
+ const isSupported = sttProvider === 'server'
+ ? audioStreamService.isSupported()
+ : browserVoiceService.isSupported();
+
// Server TTS for mobile (bypasses Safari audio restrictions)
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS({
enabled: shouldCheckOpenAIAvailability,
@@ -170,6 +183,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
isActiveRef.current = false;
processingMessageRef.current = false;
browserVoiceService.stopListening();
+ audioStreamService.stopListening();
browserVoiceService.cancelSpeech();
setStatus('idle');
setError(null);
@@ -309,11 +323,15 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
if (isActiveRef.current) {
setStatus('listening');
setError(null);
- browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
+ if (sttProvider === 'server') {
+ audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechError).catch(() => {});
+ } else {
+ browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
+ }
}
}, 1000);
}
- }, [language, conversationMode]);
+ }, [language, conversationMode, sttProvider]);
// Update the ref when handleSpeechError changes
useEffect(() => {
@@ -336,6 +354,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
// Stop listening while processing
browserVoiceService.stopListening();
+ audioStreamService.stopListening();
// Non-continuous mode: fill chat input only, do not auto-send.
if (!conversationMode) {
@@ -423,7 +442,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
setStatus('listening');
- if (isMobile) {
+ if (sttProvider === 'server') {
+ audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
+ console.error('[useBrowserVoice] Failed to restart server STT:', err);
+ });
+ } else if (isMobile) {
try {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} catch (err) {
@@ -514,7 +537,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
// Only restart listening if conversation mode is enabled
if (conversationMode) {
setStatus('listening');
- if (isMobile) {
+ if (sttProvider === 'server') {
+ audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
+ console.error('[useBrowserVoice] Failed to restart server STT after speech error:', err);
+ });
+ } else if (isMobile) {
try {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} catch (restartErr) {
@@ -546,7 +573,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
setStatus('error');
processingMessageRef.current = false;
}
- }, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode]);
+ }, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, sttProvider]);
// Handle speech recognition result
const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
@@ -589,7 +616,9 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
pendingResumeOnVisibleRef.current = false;
setStatus('listening');
try {
- if (isMobile) {
+ if (sttProvider === 'server') {
+ void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+ } else if (isMobile) {
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
@@ -605,7 +634,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
return () => {
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
- }, [conversationMode, isMobile, language]);
+ }, [conversationMode, isMobile, language, sttProvider]);
useEffect(() => {
if (typeof navigator === 'undefined') {
@@ -640,11 +669,16 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
try {
- browserVoiceService.stopListening();
- if (isMobile) {
- browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+ if (sttProvider === 'server') {
+ audioStreamService.stopListening();
+ void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
} else {
- void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+ browserVoiceService.stopListening();
+ if (isMobile) {
+ browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+ } else {
+ void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+ }
}
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
@@ -664,7 +698,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
deviceChangeRestartTimerRef.current = null;
}
};
- }, [isMobile, language, status]);
+ }, [isMobile, language, status, sttProvider]);
// Update the ref when handleSpeechResult changes
useEffect(() => {
@@ -676,6 +710,10 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
if (!isSupported) {
return false;
}
+ if (sttProvider === 'server') {
+ // getUserMedia permission is requested on startListening; nothing to prepare
+ return true;
+ }
try {
await browserVoiceService.prepareListening();
return true;
@@ -684,12 +722,12 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
setError(errorMsg);
return false;
}
- }, [isSupported]);
+ }, [isSupported, sttProvider]);
// Start voice mode
const startVoice = useCallback(async () => {
if (!isSupported) {
- setError('Browser voice not supported');
+ setError('Voice input not supported in this browser');
setStatus('error');
return;
}
@@ -704,7 +742,29 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
lastTranscriptRef.current = '';
setError(null);
setStatus('listening');
-
+
+ if (sttProvider === 'server') {
+ // Server STT: configure the service then start async recording
+ audioStreamService.configure({
+ baseURL: sttServerUrl,
+ model: sttModel,
+ language: sttLanguage || undefined,
+ silenceThresholdDb: sttSilenceThresholdDb,
+ silenceHoldMs: sttSilenceHoldMs,
+ });
+ try {
+ await audioStreamService.startListening(language, handleSpeechResult, handleSpeechError);
+ } catch (err) {
+ const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
+ console.error('[useBrowserVoice] Server STT start error:', errorMsg);
+ setError(errorMsg);
+ setStatus('error');
+ isActiveRef.current = false;
+ }
+ return;
+ }
+
+ // Browser STT
// On mobile, use sync path to ensure SpeechRecognition.start() is called
// within the same user gesture context (required by iOS Safari)
// Also unlock audio immediately for TTS playback later
@@ -742,7 +802,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
isActiveRef.current = false;
}
}
- }, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio]);
+ }, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]);
// Stop voice mode
const stopVoice = useCallback(() => {
@@ -759,6 +819,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
finalTranscriptTimerRef.current = null;
}
browserVoiceService.stopListening();
+ audioStreamService.stopListening();
browserVoiceService.cancelSpeech();
stopServerTTS(); // Also stop server TTS if playing
stopSayTTS(); // Also stop Say TTS if playing
@@ -781,6 +842,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
}
browserVoiceService.setConversationMode(false);
browserVoiceService.stopListening();
+ audioStreamService.stopListening();
browserVoiceService.cancelSpeech();
};
}, []);
diff --git a/packages/ui/src/lib/voice/audioStreamService.ts b/packages/ui/src/lib/voice/audioStreamService.ts
new file mode 100644
index 00000000..fe1d5f61
--- /dev/null
+++ b/packages/ui/src/lib/voice/audioStreamService.ts
@@ -0,0 +1,346 @@
+/**
+ * Audio Stream Service
+ *
+ * Captures microphone audio using MediaRecorder, detects utterance boundaries
+ * via an AnalyserNode-based silence detector (VAD), then POSTs each utterance
+ * as a raw audio blob to the OpenChamber server's /api/stt/transcribe endpoint.
+ *
+ * Mimics the BrowserVoiceService.startListening interface so useBrowserVoice
+ * can swap providers without changing its internal logic.
+ *
+ * @example
+ * ```typescript
+ * audioStreamService.configure({ baseURL: 'http://localhost:8001/v1', model: 'whisper-1' });
+ * audioStreamService.startListening('en', (text, isFinal) => {
+ * if (isFinal) console.log('transcript:', text);
+ * });
+ * audioStreamService.stopListening();
+ * ```
+ */
+
+export type SpeechResultCallback = (text: string, isFinal: boolean) => void;
+export type ErrorCallback = (error: string) => void;
+
+export interface AudioStreamConfig {
+ /** Base URL of the OpenAI-compatible STT server (e.g. http://localhost:8001/v1) */
+ baseURL: string;
+ /** Whisper-compatible model name */
+ model: string;
+ /** Optional BCP-47 language hint (e.g. 'en'). Empty string = auto-detect. */
+ language?: string;
+ /**
+ * Silence threshold in dB below which audio is considered silence.
+ * Lower (more negative) = only very quiet audio counts as silence.
+ * Default: -45
+ */
+ silenceThresholdDb?: number;
+ /**
+ * How long continuous silence must last (ms) before the utterance is finalised.
+ * Default: 1500
+ */
+ silenceHoldMs?: number;
+}
+
+// How often (ms) the VAD samples the analyser
+const VAD_POLL_MS = 80;
+// Minimum audio duration (ms) to bother uploading (avoids blank clips)
+const MIN_UTTERANCE_MS = 300;
+
+class AudioStreamService {
+ private stream: MediaStream | null = null;
+ private mediaRecorder: MediaRecorder | null = null;
+ private audioContext: AudioContext | null = null;
+ private analyser: AnalyserNode | null = null;
+ private vadTimer: ReturnType
| null = null;
+ private chunks: Blob[] = [];
+ private recordingStartMs = 0;
+ private isActive = false;
+ private isSpeaking = false;
+ private silenceSince: number | null = null;
+ private onResult: SpeechResultCallback | null = null;
+ private onError: ErrorCallback | null = null;
+ private lang = 'en';
+
+ // Configurable parameters
+ private cfg: Required = {
+ baseURL: '',
+ model: 'deepdml/faster-whisper-large-v3-turbo-ct2',
+ language: '',
+ silenceThresholdDb: -45,
+ silenceHoldMs: 1500,
+ };
+
+ /** Update service configuration. Can be called before or after startListening. */
+ configure(config: AudioStreamConfig): void {
+ this.cfg = {
+ silenceThresholdDb: -45,
+ silenceHoldMs: 1500,
+ language: '',
+ ...config,
+ };
+ }
+
+ /** Whether the browser supports the required APIs. */
+ isSupported(): boolean {
+ return (
+ typeof window !== 'undefined' &&
+ typeof navigator !== 'undefined' &&
+ typeof navigator.mediaDevices?.getUserMedia === 'function' &&
+ typeof window.MediaRecorder !== 'undefined' &&
+ typeof window.AudioContext !== 'undefined'
+ );
+ }
+
+ /**
+ * Start listening. Requests microphone access if not already held.
+ * Calls onResult(text, true) for each completed utterance.
+ */
+ async startListening(
+ lang: string,
+ onResult: SpeechResultCallback,
+ onError?: ErrorCallback
+ ): Promise {
+ if (this.isActive) {
+ this.stopListening();
+ }
+
+ this.lang = lang;
+ this.onResult = onResult;
+ this.onError = onError ?? null;
+ this.isActive = true;
+
+ try {
+ this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
+ } catch (err) {
+ this.isActive = false;
+ const msg = err instanceof Error ? err.message : 'Microphone access denied';
+ onError?.(msg);
+ return;
+ }
+
+ this._setupAudioContext();
+ this._startRecorder();
+ this._startVAD();
+ }
+
+ /** Stop listening and clean up all resources. */
+ stopListening(): void {
+ this.isActive = false;
+ this._stopVAD();
+ this._stopRecorder();
+ this._teardownAudioContext();
+ this._releaseStream();
+ this.chunks = [];
+ this.isSpeaking = false;
+ this.silenceSince = null;
+ this.onResult = null;
+ this.onError = null;
+ }
+
+ /** Whether currently listening. */
+ getIsListening(): boolean {
+ return this.isActive;
+ }
+
+ // ── Private helpers ──────────────────────────────────────────────────────
+
+ private _setupAudioContext(): void {
+ if (!this.stream) return;
+ const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
+ this.audioContext = new AudioContextClass();
+ const source = this.audioContext.createMediaStreamSource(this.stream);
+ this.analyser = this.audioContext.createAnalyser();
+ this.analyser.fftSize = 512;
+ source.connect(this.analyser);
+ }
+
+ private _teardownAudioContext(): void {
+ try {
+ this.audioContext?.close();
+ } catch {
+ // ignore
+ }
+ this.audioContext = null;
+ this.analyser = null;
+ }
+
+ private _startRecorder(): void {
+ if (!this.stream) return;
+
+ const mimeType = this._pickMimeType();
+ const options: MediaRecorderOptions = {};
+ if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
+ options.mimeType = mimeType;
+ }
+
+ this.mediaRecorder = new MediaRecorder(this.stream, options);
+ this.chunks = [];
+ this.recordingStartMs = Date.now();
+
+ this.mediaRecorder.ondataavailable = (e) => {
+ if (e.data && e.data.size > 0) {
+ this.chunks.push(e.data);
+ }
+ };
+
+ this.mediaRecorder.onstop = () => {
+ const blobs = this.chunks.splice(0);
+ const durationMs = Date.now() - this.recordingStartMs;
+ if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) return;
+
+ const mType = blobs[0].type || mimeType || 'audio/webm';
+ const blob = new Blob(blobs, { type: mType });
+ void this._upload(blob, mType);
+ };
+
+ // Collect data every 250 ms so we don't lose the tail on stop()
+ this.mediaRecorder.start(250);
+ }
+
+ private _stopRecorder(): void {
+ if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
+ try {
+ this.mediaRecorder.stop();
+ } catch {
+ // ignore
+ }
+ }
+ this.mediaRecorder = null;
+ }
+
+ private _releaseStream(): void {
+ if (this.stream) {
+ this.stream.getTracks().forEach((t) => t.stop());
+ this.stream = null;
+ }
+ }
+
+ private _startVAD(): void {
+ this._stopVAD();
+ this.silenceSince = null;
+ this.isSpeaking = false;
+
+ this.vadTimer = setInterval(() => {
+ if (!this.isActive || !this.analyser) return;
+ const db = this._getRmsDb();
+ const isSilent = db < this.cfg.silenceThresholdDb;
+
+ if (!isSilent) {
+ // Audio detected
+ this.silenceSince = null;
+ if (!this.isSpeaking) {
+ this.isSpeaking = true;
+ // Restart recorder to capture from the start of speech
+ if (this.mediaRecorder?.state === 'recording') {
+ this.recordingStartMs = Date.now();
+ }
+ }
+ } else {
+ // Silence detected
+ if (this.isSpeaking) {
+ if (this.silenceSince === null) {
+ this.silenceSince = Date.now();
+ } else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
+ // End of utterance — stop recorder (triggers onstop → upload)
+ this.isSpeaking = false;
+ this.silenceSince = null;
+ this._finaliseUtterance();
+ }
+ }
+ }
+ }, VAD_POLL_MS);
+ }
+
+ private _stopVAD(): void {
+ if (this.vadTimer !== null) {
+ clearInterval(this.vadTimer);
+ this.vadTimer = null;
+ }
+ }
+
+ /** Stop the current recorder to flush the utterance, then restart it. */
+ private _finaliseUtterance(): void {
+ if (!this.isActive) return;
+ if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
+ this.mediaRecorder.stop();
+ }
+ // Restart recorder for the next utterance after a short delay
+ // (MediaRecorder.onstop fires asynchronously; we wait for it to complete)
+ setTimeout(() => {
+ if (this.isActive && this.stream) {
+ this._startRecorder();
+ }
+ }, 100);
+ }
+
+ /** Compute RMS of current analyser frame in dBFS. */
+ private _getRmsDb(): number {
+ if (!this.analyser) return -Infinity;
+ const buf = new Float32Array(this.analyser.fftSize);
+ this.analyser.getFloatTimeDomainData(buf);
+ let sumSq = 0;
+ for (const s of buf) sumSq += s * s;
+ const rms = Math.sqrt(sumSq / buf.length);
+ return rms === 0 ? -Infinity : 20 * Math.log10(rms);
+ }
+
+ /** POST utterance blob to server, call onResult with transcript. */
+ private async _upload(blob: Blob, mimeType: string): Promise {
+ if (!this.onResult) return;
+
+ try {
+ const headers: Record = {
+ 'Content-Type': mimeType,
+ 'X-Base-URL': this.cfg.baseURL,
+ 'X-Model': this.cfg.model,
+ };
+ if (this.cfg.language) {
+ headers['X-Language'] = this.cfg.language;
+ } else if (this.lang && this.lang !== 'auto') {
+ // Use BCP-47 base language code (e.g. 'en' from 'en-US')
+ const baseLang = this.lang.split('-')[0];
+ headers['X-Language'] = baseLang;
+ }
+
+ const response = await fetch('/api/stt/transcribe', {
+ method: 'POST',
+ headers,
+ body: blob,
+ });
+
+ if (!response.ok) {
+ const errData = await response.json().catch(() => ({ error: 'Unknown error' }));
+ throw new Error(errData.error ?? `HTTP ${response.status}`);
+ }
+
+ const data = await response.json();
+ const transcript: string = (data.transcript ?? '').trim();
+ if (transcript) {
+ this.onResult(transcript, true);
+ }
+ } catch (err) {
+ if (!this.isActive) return; // Stopped — ignore
+ const msg = err instanceof Error ? err.message : 'Transcription upload failed';
+ console.error('[AudioStreamService] Upload error:', msg);
+ this.onError?.(msg);
+ }
+ }
+
+ /** Pick the best supported MIME type for MediaRecorder. */
+ private _pickMimeType(): string {
+ const candidates = [
+ 'audio/webm;codecs=opus',
+ 'audio/webm',
+ 'audio/ogg;codecs=opus',
+ 'audio/ogg',
+ 'audio/mp4',
+ ];
+ if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
+ return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
+ }
+ return '';
+ }
+}
+
+export const audioStreamService = new AudioStreamService();
+export { AudioStreamService };
diff --git a/packages/ui/src/stores/useConfigStore.ts b/packages/ui/src/stores/useConfigStore.ts
index f6614888..e5214e4f 100644
--- a/packages/ui/src/stores/useConfigStore.ts
+++ b/packages/ui/src/stores/useConfigStore.ts
@@ -477,6 +477,15 @@ interface ConfigStore {
browserVoice: string;
openaiVoice: string;
openaiApiKey: string;
+ openaiCompatibleUrl: string;
+ openaiCompatibleVoice: string;
+ // STT (speech-to-text) settings
+ sttProvider: 'browser' | 'server';
+ sttServerUrl: string;
+ sttModel: string;
+ sttLanguage: string;
+ sttSilenceThresholdDb: number;
+ sttSilenceHoldMs: number;
showMessageTTSButtons: boolean;
voiceModeEnabled: boolean;
// Summarization settings
@@ -491,6 +500,14 @@ interface ConfigStore {
setBrowserVoice: (voice: string) => void;
setOpenaiVoice: (voice: string) => void;
setOpenaiApiKey: (apiKey: string) => void;
+ setOpenaiCompatibleUrl: (url: string) => void;
+ setOpenaiCompatibleVoice: (voice: string) => void;
+ setSttProvider: (provider: 'browser' | 'server') => void;
+ setSttServerUrl: (url: string) => void;
+ setSttModel: (model: string) => void;
+ setSttLanguage: (lang: string) => void;
+ setSttSilenceThresholdDb: (db: number) => void;
+ setSttSilenceHoldMs: (ms: number) => void;
setShowMessageTTSButtons: (show: boolean) => void;
setVoiceModeEnabled: (enabled: boolean) => void;
setSummarizeMessageTTS: (enabled: boolean) => void;
@@ -635,6 +652,71 @@ export const useConfigStore = create()(
}
return '';
})(),
+ // OpenAI-compatible custom server URL
+ openaiCompatibleUrl: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('openaiCompatibleUrl');
+ if (saved) return saved;
+ }
+ return '';
+ })(),
+ // OpenAI-compatible custom server voice
+ openaiCompatibleVoice: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('openaiCompatibleVoice');
+ if (saved) return saved;
+ }
+ return 'af_sky';
+ })(),
+ // STT provider: 'browser' (Web Speech API) or 'server' (OpenAI-compat)
+ sttProvider: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('sttProvider');
+ if (saved === 'browser' || saved === 'server') return saved;
+ }
+ return 'browser' as const;
+ })(),
+ sttServerUrl: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('sttServerUrl');
+ if (saved) return saved;
+ }
+ return 'http://localhost:8001/v1';
+ })(),
+ sttModel: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('sttModel');
+ if (saved) return saved;
+ }
+ return 'deepdml/faster-whisper-large-v3-turbo-ct2';
+ })(),
+ sttLanguage: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('sttLanguage');
+ if (saved !== null) return saved;
+ }
+ return '';
+ })(),
+ sttSilenceThresholdDb: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('sttSilenceThresholdDb');
+ if (saved) {
+ const parsed = parseFloat(saved);
+ if (!isNaN(parsed)) return parsed;
+ }
+ }
+ return -45;
+ })(),
+ sttSilenceHoldMs: (() => {
+ if (typeof window !== 'undefined') {
+ const saved = localStorage.getItem('sttSilenceHoldMs');
+ if (saved) {
+ const parsed = parseInt(saved, 10);
+ if (!isNaN(parsed)) return parsed;
+ }
+ }
+ return 1500;
+ })(),
// Show TTS buttons on messages - disabled by default until user enables it
showMessageTTSButtons: (() => {
if (typeof window !== 'undefined') {
@@ -1662,6 +1744,62 @@ export const useConfigStore = create()(
}
},
+ setOpenaiCompatibleUrl: (url: string) => {
+ set({ openaiCompatibleUrl: url });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('openaiCompatibleUrl', url);
+ }
+ },
+
+ setOpenaiCompatibleVoice: (voice: string) => {
+ set({ openaiCompatibleVoice: voice });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('openaiCompatibleVoice', voice);
+ }
+ },
+
+ setSttProvider: (provider: 'browser' | 'server') => {
+ set({ sttProvider: provider });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('sttProvider', provider);
+ }
+ },
+
+ setSttServerUrl: (url: string) => {
+ set({ sttServerUrl: url });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('sttServerUrl', url);
+ }
+ },
+
+ setSttModel: (model: string) => {
+ set({ sttModel: model });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('sttModel', model);
+ }
+ },
+
+ setSttLanguage: (lang: string) => {
+ set({ sttLanguage: lang });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('sttLanguage', lang);
+ }
+ },
+
+ setSttSilenceThresholdDb: (db: number) => {
+ set({ sttSilenceThresholdDb: db });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('sttSilenceThresholdDb', String(db));
+ }
+ },
+
+ setSttSilenceHoldMs: (ms: number) => {
+ set({ sttSilenceHoldMs: ms });
+ if (typeof window !== 'undefined') {
+ localStorage.setItem('sttSilenceHoldMs', String(ms));
+ }
+ },
+
setShowMessageTTSButtons: (show: boolean) => {
set({ showMessageTTSButtons: show });
if (typeof window !== 'undefined') {
diff --git a/packages/web/server/lib/tts/routes.js b/packages/web/server/lib/tts/routes.js
index 170c80ac..217e0d96 100644
--- a/packages/web/server/lib/tts/routes.js
+++ b/packages/web/server/lib/tts/routes.js
@@ -1,3 +1,5 @@
+import express from 'express';
+
export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
let ttsModulePromise = null;
const getTtsModule = async () => {
@@ -222,4 +224,54 @@ export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
});
}
});
+
+ // Server-side STT: receive raw audio, proxy to OpenAI-compatible transcription endpoint
+ app.post(
+ '/api/stt/transcribe',
+ express.raw({ type: (req) => (req.headers['content-type'] || '').startsWith('audio/'), limit: '20mb' }),
+ async (req, res) => {
+ try {
+ const { transcribeAudio } = await import('./stt.js');
+
+ const mimeType = (req.headers['content-type'] || 'audio/webm').split(',')[0].trim();
+ const baseURL = req.headers['x-base-url'];
+ const model = req.headers['x-model'] || 'deepdml/faster-whisper-large-v3-turbo-ct2';
+ const language = req.headers['x-language'] || undefined;
+
+ if (!req.body || !Buffer.isBuffer(req.body) || req.body.length === 0) {
+ return res.status(400).json({ error: 'Audio data is required' });
+ }
+
+ if (!baseURL) {
+ return res.status(400).json({ error: 'X-Base-URL header is required' });
+ }
+
+ console.log('[STT] Transcribing audio:', {
+ bytes: req.body.length,
+ mimeType,
+ model,
+ baseURL,
+ language,
+ });
+
+ const transcript = await transcribeAudio({
+ audioBuffer: req.body,
+ mimeType,
+ model,
+ baseURL,
+ language,
+ });
+
+ console.log('[STT] Transcript:', transcript?.slice(0, 120));
+ res.json({ transcript: transcript ?? '' });
+ } catch (error) {
+ console.error('[STT] Error:', error);
+ if (!res.headersSent) {
+ res.status(500).json({
+ error: error instanceof Error ? error.message : 'Transcription failed',
+ });
+ }
+ }
+ }
+ );
}
diff --git a/packages/web/server/lib/tts/stt.js b/packages/web/server/lib/tts/stt.js
new file mode 100644
index 00000000..a947ae6a
--- /dev/null
+++ b/packages/web/server/lib/tts/stt.js
@@ -0,0 +1,66 @@
+/**
+ * Server-side Speech-to-Text Service
+ *
+ * Proxies audio to any OpenAI-compatible transcription endpoint
+ * (e.g. faster-whisper, whisper.cpp) using the OpenAI Node SDK.
+ */
+
+import OpenAI, { toFile } from 'openai';
+
+/**
+ * Transcribe an audio buffer via an OpenAI-compatible /v1/audio/transcriptions endpoint.
+ *
+ * @param {object} opts
+ * @param {Buffer} opts.audioBuffer - Raw audio bytes
+ * @param {string} opts.mimeType - MIME type of the audio (e.g. 'audio/webm')
+ * @param {string} opts.model - Model name accepted by the remote server
+ * @param {string} [opts.baseURL] - Base URL of the compatible server (including /v1)
+ * @param {string} [opts.language] - Optional BCP-47 language hint (e.g. 'en')
+ * @returns {Promise} Transcribed text
+ */
+export async function transcribeAudio({ audioBuffer, mimeType, model, baseURL, language }) {
+ const clientOpts = {
+ apiKey: process.env.OPENAI_API_KEY || 'not-required',
+ };
+ if (baseURL) {
+ clientOpts.baseURL = baseURL;
+ }
+
+ const client = new OpenAI(clientOpts);
+
+ // Derive a sensible filename extension from the MIME type so the server
+ // can infer the codec when it isn't explicit in the stream header.
+ const ext = mimeTypeToExt(mimeType);
+ const filename = `audio.${ext}`;
+
+ const file = await toFile(audioBuffer, filename, { type: mimeType });
+
+ const result = await client.audio.transcriptions.create({
+ file,
+ model,
+ response_format: 'json',
+ ...(language ? { language } : {}),
+ });
+
+ return result.text ?? '';
+}
+
+/**
+ * Map a MIME type to a file extension understood by Whisper servers.
+ * @param {string} mimeType
+ * @returns {string}
+ */
+function mimeTypeToExt(mimeType) {
+ const type = (mimeType || '').split(';')[0].trim().toLowerCase();
+ const map = {
+ 'audio/webm': 'webm',
+ 'audio/ogg': 'ogg',
+ 'audio/wav': 'wav',
+ 'audio/wave': 'wav',
+ 'audio/mpeg': 'mp3',
+ 'audio/mp4': 'mp4',
+ 'audio/mp3': 'mp3',
+ 'audio/flac': 'flac',
+ };
+ return map[type] ?? 'webm';
+}