feat(stt): add server-side STT provider via OpenAI-compatible Whisper endpoint (#860)

Co-authored-by: Alexander Busse <alex@ableph.net>

packages/ui/src/components/sections/openchamber/VoiceSettings.tsx

@@ -16,6 +16,7 @@ import { NumberInput } from '@/components/ui/number-input';
 import { RiPlayLine, RiStopLine, RiCloseLine, RiAppleLine, RiInformationLine } from '@remixicon/react';
 import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
 import { browserVoiceService } from '@/lib/voice/browserVoiceService';
+import { audioStreamService } from '@/lib/voice/audioStreamService';
 import { cn } from '@/lib/utils';
 
 const LANGUAGE_OPTIONS = [
@@ -71,6 +72,19 @@ export const VoiceSettings: React.FC = () => {
     const openaiApiKey = useConfigStore((state) => state.openaiApiKey);
     const setOpenaiApiKey = useConfigStore((state) => state.setOpenaiApiKey);
     const showMessageTTSButtons = useConfigStore((state) => state.showMessageTTSButtons);
+    // STT settings
+    const sttProvider = useConfigStore((state) => state.sttProvider);
+    const setSttProvider = useConfigStore((state) => state.setSttProvider);
+    const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
+    const setSttServerUrl = useConfigStore((state) => state.setSttServerUrl);
+    const sttModel = useConfigStore((state) => state.sttModel);
+    const setSttModel = useConfigStore((state) => state.setSttModel);
+    const sttLanguage = useConfigStore((state) => state.sttLanguage);
+    const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
+    const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
+    const setSttSilenceThresholdDb = useConfigStore((state) => state.setSttSilenceThresholdDb);
+    const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
+    const setSttSilenceHoldMs = useConfigStore((state) => state.setSttSilenceHoldMs);
     const setShowMessageTTSButtons = useConfigStore((state) => state.setShowMessageTTSButtons);
     const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
     const setVoiceModeEnabled = useConfigStore((state) => state.setVoiceModeEnabled);
@@ -563,6 +577,146 @@ export const VoiceSettings: React.FC = () => {
                 </section>
             </div>
 
+            {/* Speech Recognition */}
+            {voiceModeEnabled && (
+                <div className="mb-8">
+                    <div className="mb-1 px-1">
+                        <h3 className="typography-ui-header font-medium text-foreground">
+                            Speech Recognition
+                        </h3>
+                    </div>
+
+                    <section className="px-2 pb-2 pt-0 space-y-0">
+                        <div className="pb-1.5 pt-0.5">
+                            <div className="flex min-w-0 flex-col gap-1.5">
+                                <div className="flex items-center gap-1.5">
+                                    <span className="typography-ui-label text-foreground">Provider</span>
+                                    <Tooltip delayDuration={1000}>
+                                        <TooltipTrigger asChild>
+                                            <RiInformationLine className="h-3.5 w-3.5 text-muted-foreground/60 cursor-help" />
+                                        </TooltipTrigger>
+                                        <TooltipContent sideOffset={8} className="max-w-xs">
+                                            <ul className="space-y-1">
+                                                <li><strong>Browser:</strong> Web Speech API (Chrome/Edge). Free, no setup.</li>
+                                                <li><strong>Server:</strong> OpenAI-compatible Whisper server. Better accuracy, any language.</li>
+                                            </ul>
+                                        </TooltipContent>
+                                    </Tooltip>
+                                </div>
+                                <div className="flex flex-wrap items-center gap-1">
+                                    <Button
+                                        variant="outline"
+                                        size="xs"
+                                        onClick={() => setSttProvider('browser')}
+                                        className={cn(
+                                            '!font-normal',
+                                            sttProvider === 'browser'
+                                                ? 'border-[var(--primary-base)] text-[var(--primary-base)] bg-[var(--primary-base)]/10 hover:text-[var(--primary-base)]'
+                                                : 'text-foreground'
+                                        )}
+                                    >
+                                        Browser
+                                    </Button>
+                                    <Button
+                                        variant="outline"
+                                        size="xs"
+                                        onClick={() => setSttProvider('server')}
+                                        className={cn(
+                                            '!font-normal',
+                                            sttProvider === 'server'
+                                                ? 'border-[var(--primary-base)] text-[var(--primary-base)] bg-[var(--primary-base)]/10 hover:text-[var(--primary-base)]'
+                                                : 'text-foreground'
+                                        )}
+                                    >
+                                        Server
+                                    </Button>
+                                </div>
+                            </div>
+                        </div>
+
+                        {sttProvider === 'server' && (
+                            <div className="py-1.5 space-y-2">
+                                {!audioStreamService.isSupported() && (
+                                    <p className="typography-meta text-[var(--status-error)]">
+                                        MediaRecorder or AudioContext is not available in this browser. Server STT may not work.
+                                    </p>
+                                )}
+                                <div>
+                                    <span className={cn("typography-ui-label text-foreground", !sttServerUrl.trim() && "text-[var(--status-error)]")}>
+                                        Server URL
+                                    </span>
+                                    <span className="typography-meta ml-2 text-muted-foreground">
+                                        Base URL of the Whisper-compatible server
+                                    </span>
+                                    <div className="relative mt-1.5 max-w-xs">
+                                        <input
+                                            type="text"
+                                            value={sttServerUrl}
+                                            onChange={(e) => setSttServerUrl(e.target.value)}
+                                            placeholder="http://localhost:8001/v1"
+                                            className="w-full h-7 rounded-lg border border-input bg-transparent px-2 typography-ui-label text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-primary/50 focus:border-primary/70"
+                                        />
+                                        {sttServerUrl && (
+                                            <button
+                                                type="button"
+                                                onClick={() => setSttServerUrl('')}
+                                                className="absolute right-2 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground"
+                                            >
+                                                <RiCloseLine className="w-3.5 h-3.5" />
+                                            </button>
+                                        )}
+                                    </div>
+                                </div>
+                                <div>
+                                    <span className="typography-ui-label text-foreground">Model</span>
+                                    <div className="relative mt-1.5 max-w-xs">
+                                        <input
+                                            type="text"
+                                            value={sttModel}
+                                            onChange={(e) => setSttModel(e.target.value)}
+                                            placeholder="deepdml/faster-whisper-large-v3-turbo-ct2"
+                                            className="w-full h-7 rounded-lg border border-input bg-transparent px-2 typography-ui-label text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-primary/50 focus:border-primary/70"
+                                        />
+                                    </div>
+                                </div>
+                                <div>
+                                    <span className="typography-ui-label text-foreground">Language</span>
+                                    <span className="typography-meta ml-2 text-muted-foreground">
+                                        BCP-47 code (e.g. en, fr). Leave blank for auto-detect.
+                                    </span>
+                                    <div className="relative mt-1.5 max-w-[8rem]">
+                                        <input
+                                            type="text"
+                                            value={sttLanguage}
+                                            onChange={(e) => setSttLanguage(e.target.value)}
+                                            placeholder="auto"
+                                            className="w-full h-7 rounded-lg border border-input bg-transparent px-2 typography-ui-label text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-primary/50 focus:border-primary/70"
+                                        />
+                                    </div>
+                                </div>
+                                <div className="flex items-center gap-8 py-0.5">
+                                    <span className="typography-ui-label text-foreground sm:w-56 shrink-0">Silence Threshold</span>
+                                    <div className="flex items-center gap-2 w-fit">
+                                        {!isMobile && <input type="range" min={-60} max={-20} step={1} value={sttSilenceThresholdDb} onChange={(e) => setSttSilenceThresholdDb(Number(e.target.value))} className={sliderClass} />}
+                                        <span className="typography-ui-label text-foreground tabular-nums min-w-[3.5rem] text-right">
+                                            {sttSilenceThresholdDb} dB
+                                        </span>
+                                    </div>
+                                </div>
+                                <div className="flex items-center gap-8 py-0.5">
+                                    <span className="typography-ui-label text-foreground sm:w-56 shrink-0">Silence Hold</span>
+                                    <div className="flex items-center gap-2 w-fit">
+                                        {!isMobile && <input type="range" min={500} max={3000} step={100} value={sttSilenceHoldMs} onChange={(e) => setSttSilenceHoldMs(Number(e.target.value))} className={sliderClass} />}
+                                        <NumberInput value={sttSilenceHoldMs} onValueChange={setSttSilenceHoldMs} min={500} max={3000} step={100} className="w-20 tabular-nums" />
+                                        <span className="typography-meta text-muted-foreground">ms</span>
+                                    </div>
+                                </div>
+                            </div>
+                        )}
+                    </section>
+                </div>
+            )}
+
             {/* Playback & Summarization */}
             <div className="mb-8">
                 <div className="mb-1 px-1">

packages/ui/src/hooks/useBrowserVoice.ts

@@ -27,6 +27,7 @@
 
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { browserVoiceService } from '@/lib/voice/browserVoiceService';
+import { audioStreamService } from '@/lib/voice/audioStreamService';
 import { useSessionUIStore } from '@/sync/session-ui-store';
 import { useInputStore } from '@/sync/input-store';
 import { getSyncMessages, getSyncParts } from '@/sync/sync-refs';
@@ -109,8 +110,6 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
     return false;
   });
   
-  const isSupported = browserVoiceService.isSupported();
-  
   // Mobile detection
   const isMobile = useMemo(() => {
     if (typeof navigator === 'undefined') return false;
@@ -143,12 +142,26 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
   const sayVoice = useConfigStore((state) => state.sayVoice);
   const browserVoice = useConfigStore((state) => state.browserVoice);
   const openaiVoice = useConfigStore((state) => state.openaiVoice);
+  const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
+  const openaiCompatibleUrl = useConfigStore((state) => state.openaiCompatibleUrl);
   const summarizeVoiceConversation = useConfigStore((state) => state.summarizeVoiceConversation);
   const summarizeCharacterThreshold = useConfigStore((state) => state.summarizeCharacterThreshold);
 
   const shouldCheckOpenAIAvailability = voiceModeEnabled && voiceProvider === 'openai';
   const shouldCheckSayAvailability = voiceModeEnabled && voiceProvider === 'say';
 
+  // STT provider config
+  const sttProvider = useConfigStore((state) => state.sttProvider);
+  const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
+  const sttModel = useConfigStore((state) => state.sttModel);
+  const sttLanguage = useConfigStore((state) => state.sttLanguage);
+  const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
+  const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
+
+  const isSupported = sttProvider === 'server'
+    ? audioStreamService.isSupported()
+    : browserVoiceService.isSupported();
+
   // Server TTS for mobile (bypasses Safari audio restrictions)
   const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS({
     enabled: shouldCheckOpenAIAvailability,
@@ -170,6 +183,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
         isActiveRef.current = false;
         processingMessageRef.current = false;
         browserVoiceService.stopListening();
+        audioStreamService.stopListening();
         browserVoiceService.cancelSpeech();
         setStatus('idle');
         setError(null);
@@ -309,11 +323,15 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
         if (isActiveRef.current) {
           setStatus('listening');
           setError(null);
-          browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
+          if (sttProvider === 'server') {
+            audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechError).catch(() => {});
+          } else {
+            browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
+          }
         }
       }, 1000);
     }
-  }, [language, conversationMode]);
+  }, [language, conversationMode, sttProvider]);
 
   // Update the ref when handleSpeechError changes
   useEffect(() => {
@@ -336,6 +354,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
 
     // Stop listening while processing
     browserVoiceService.stopListening();
+    audioStreamService.stopListening();
 
     // Non-continuous mode: fill chat input only, do not auto-send.
     if (!conversationMode) {
@@ -423,7 +442,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
                   }
 
                   setStatus('listening');
-                  if (isMobile) {
+                  if (sttProvider === 'server') {
+                    audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
+                      console.error('[useBrowserVoice] Failed to restart server STT:', err);
+                    });
+                  } else if (isMobile) {
                     try {
                       browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
                     } catch (err) {
@@ -514,7 +537,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
               // Only restart listening if conversation mode is enabled
               if (conversationMode) {
                 setStatus('listening');
-                if (isMobile) {
+                if (sttProvider === 'server') {
+                  audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
+                    console.error('[useBrowserVoice] Failed to restart server STT after speech error:', err);
+                  });
+                } else if (isMobile) {
                   try {
                     browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
                   } catch (restartErr) {
@@ -546,7 +573,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
       setStatus('error');
       processingMessageRef.current = false;
     }
-  }, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode]);
+  }, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, sttProvider]);
 
   // Handle speech recognition result
   const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
@@ -589,7 +616,9 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
       pendingResumeOnVisibleRef.current = false;
       setStatus('listening');
       try {
-        if (isMobile) {
+        if (sttProvider === 'server') {
+          void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+        } else if (isMobile) {
           browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
         } else {
           browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
@@ -605,7 +634,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
     return () => {
       document.removeEventListener('visibilitychange', handleVisibilityChange);
     };
-  }, [conversationMode, isMobile, language]);
+  }, [conversationMode, isMobile, language, sttProvider]);
 
   useEffect(() => {
     if (typeof navigator === 'undefined') {
@@ -640,11 +669,16 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
         }
 
         try {
-          browserVoiceService.stopListening();
+          if (sttProvider === 'server') {
-          if (isMobile) {
+            audioStreamService.stopListening();
-            browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+            void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
           } else {
-            void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+            browserVoiceService.stopListening();
+            if (isMobile) {
+              browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+            } else {
+              void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
+            }
           }
         } catch (err) {
           const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
@@ -664,7 +698,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
         deviceChangeRestartTimerRef.current = null;
       }
     };
-  }, [isMobile, language, status]);
+  }, [isMobile, language, status, sttProvider]);
 
   // Update the ref when handleSpeechResult changes
   useEffect(() => {
@@ -676,6 +710,10 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
     if (!isSupported) {
       return false;
     }
+    if (sttProvider === 'server') {
+      // getUserMedia permission is requested on startListening; nothing to prepare
+      return true;
+    }
     try {
       await browserVoiceService.prepareListening();
       return true;
@@ -684,12 +722,12 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
       setError(errorMsg);
       return false;
     }
-  }, [isSupported]);
+  }, [isSupported, sttProvider]);
 
   // Start voice mode
   const startVoice = useCallback(async () => {
     if (!isSupported) {
-      setError('Browser voice not supported');
+      setError('Voice input not supported in this browser');
       setStatus('error');
       return;
     }
@@ -704,7 +742,29 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
     lastTranscriptRef.current = '';
     setError(null);
     setStatus('listening');
-    
+
+    if (sttProvider === 'server') {
+      // Server STT: configure the service then start async recording
+      audioStreamService.configure({
+        baseURL: sttServerUrl,
+        model: sttModel,
+        language: sttLanguage || undefined,
+        silenceThresholdDb: sttSilenceThresholdDb,
+        silenceHoldMs: sttSilenceHoldMs,
+      });
+      try {
+        await audioStreamService.startListening(language, handleSpeechResult, handleSpeechError);
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
+        console.error('[useBrowserVoice] Server STT start error:', errorMsg);
+        setError(errorMsg);
+        setStatus('error');
+        isActiveRef.current = false;
+      }
+      return;
+    }
+
+    // Browser STT
     // On mobile, use sync path to ensure SpeechRecognition.start() is called
     // within the same user gesture context (required by iOS Safari)
     // Also unlock audio immediately for TTS playback later
@@ -742,7 +802,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
         isActiveRef.current = false;
       }
     }
-  }, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio]);
+  }, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]);
 
   // Stop voice mode
   const stopVoice = useCallback(() => {
@@ -759,6 +819,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
       finalTranscriptTimerRef.current = null;
     }
     browserVoiceService.stopListening();
+    audioStreamService.stopListening();
     browserVoiceService.cancelSpeech();
     stopServerTTS(); // Also stop server TTS if playing
     stopSayTTS(); // Also stop Say TTS if playing
@@ -781,6 +842,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
       }
       browserVoiceService.setConversationMode(false);
       browserVoiceService.stopListening();
+      audioStreamService.stopListening();
       browserVoiceService.cancelSpeech();
     };
   }, []);

packages/ui/src/lib/voice/audioStreamService.ts

@@ -0,0 +1,346 @@
+/**
+ * Audio Stream Service
+ *
+ * Captures microphone audio using MediaRecorder, detects utterance boundaries
+ * via an AnalyserNode-based silence detector (VAD), then POSTs each utterance
+ * as a raw audio blob to the OpenChamber server's /api/stt/transcribe endpoint.
+ *
+ * Mimics the BrowserVoiceService.startListening interface so useBrowserVoice
+ * can swap providers without changing its internal logic.
+ *
+ * @example
+ * ```typescript
+ * audioStreamService.configure({ baseURL: 'http://localhost:8001/v1', model: 'whisper-1' });
+ * audioStreamService.startListening('en', (text, isFinal) => {
+ *   if (isFinal) console.log('transcript:', text);
+ * });
+ * audioStreamService.stopListening();
+ * ```
+ */
+
+export type SpeechResultCallback = (text: string, isFinal: boolean) => void;
+export type ErrorCallback = (error: string) => void;
+
+export interface AudioStreamConfig {
+  /** Base URL of the OpenAI-compatible STT server (e.g. http://localhost:8001/v1) */
+  baseURL: string;
+  /** Whisper-compatible model name */
+  model: string;
+  /** Optional BCP-47 language hint (e.g. 'en'). Empty string = auto-detect. */
+  language?: string;
+  /**
+   * Silence threshold in dB below which audio is considered silence.
+   * Lower (more negative) = only very quiet audio counts as silence.
+   * Default: -45
+   */
+  silenceThresholdDb?: number;
+  /**
+   * How long continuous silence must last (ms) before the utterance is finalised.
+   * Default: 1500
+   */
+  silenceHoldMs?: number;
+}
+
+// How often (ms) the VAD samples the analyser
+const VAD_POLL_MS = 80;
+// Minimum audio duration (ms) to bother uploading (avoids blank clips)
+const MIN_UTTERANCE_MS = 300;
+
+class AudioStreamService {
+  private stream: MediaStream | null = null;
+  private mediaRecorder: MediaRecorder | null = null;
+  private audioContext: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private vadTimer: ReturnType<typeof setInterval> | null = null;
+  private chunks: Blob[] = [];
+  private recordingStartMs = 0;
+  private isActive = false;
+  private isSpeaking = false;
+  private silenceSince: number | null = null;
+  private onResult: SpeechResultCallback | null = null;
+  private onError: ErrorCallback | null = null;
+  private lang = 'en';
+
+  // Configurable parameters
+  private cfg: Required<AudioStreamConfig> = {
+    baseURL: '',
+    model: 'deepdml/faster-whisper-large-v3-turbo-ct2',
+    language: '',
+    silenceThresholdDb: -45,
+    silenceHoldMs: 1500,
+  };
+
+  /** Update service configuration. Can be called before or after startListening. */
+  configure(config: AudioStreamConfig): void {
+    this.cfg = {
+      silenceThresholdDb: -45,
+      silenceHoldMs: 1500,
+      language: '',
+      ...config,
+    };
+  }
+
+  /** Whether the browser supports the required APIs. */
+  isSupported(): boolean {
+    return (
+      typeof window !== 'undefined' &&
+      typeof navigator !== 'undefined' &&
+      typeof navigator.mediaDevices?.getUserMedia === 'function' &&
+      typeof window.MediaRecorder !== 'undefined' &&
+      typeof window.AudioContext !== 'undefined'
+    );
+  }
+
+  /**
+   * Start listening. Requests microphone access if not already held.
+   * Calls onResult(text, true) for each completed utterance.
+   */
+  async startListening(
+    lang: string,
+    onResult: SpeechResultCallback,
+    onError?: ErrorCallback
+  ): Promise<void> {
+    if (this.isActive) {
+      this.stopListening();
+    }
+
+    this.lang = lang;
+    this.onResult = onResult;
+    this.onError = onError ?? null;
+    this.isActive = true;
+
+    try {
+      this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
+    } catch (err) {
+      this.isActive = false;
+      const msg = err instanceof Error ? err.message : 'Microphone access denied';
+      onError?.(msg);
+      return;
+    }
+
+    this._setupAudioContext();
+    this._startRecorder();
+    this._startVAD();
+  }
+
+  /** Stop listening and clean up all resources. */
+  stopListening(): void {
+    this.isActive = false;
+    this._stopVAD();
+    this._stopRecorder();
+    this._teardownAudioContext();
+    this._releaseStream();
+    this.chunks = [];
+    this.isSpeaking = false;
+    this.silenceSince = null;
+    this.onResult = null;
+    this.onError = null;
+  }
+
+  /** Whether currently listening. */
+  getIsListening(): boolean {
+    return this.isActive;
+  }
+
+  // ── Private helpers ──────────────────────────────────────────────────────
+
+  private _setupAudioContext(): void {
+    if (!this.stream) return;
+    const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
+    this.audioContext = new AudioContextClass();
+    const source = this.audioContext.createMediaStreamSource(this.stream);
+    this.analyser = this.audioContext.createAnalyser();
+    this.analyser.fftSize = 512;
+    source.connect(this.analyser);
+  }
+
+  private _teardownAudioContext(): void {
+    try {
+      this.audioContext?.close();
+    } catch {
+      // ignore
+    }
+    this.audioContext = null;
+    this.analyser = null;
+  }
+
+  private _startRecorder(): void {
+    if (!this.stream) return;
+
+    const mimeType = this._pickMimeType();
+    const options: MediaRecorderOptions = {};
+    if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
+      options.mimeType = mimeType;
+    }
+
+    this.mediaRecorder = new MediaRecorder(this.stream, options);
+    this.chunks = [];
+    this.recordingStartMs = Date.now();
+
+    this.mediaRecorder.ondataavailable = (e) => {
+      if (e.data && e.data.size > 0) {
+        this.chunks.push(e.data);
+      }
+    };
+
+    this.mediaRecorder.onstop = () => {
+      const blobs = this.chunks.splice(0);
+      const durationMs = Date.now() - this.recordingStartMs;
+      if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) return;
+
+      const mType = blobs[0].type || mimeType || 'audio/webm';
+      const blob = new Blob(blobs, { type: mType });
+      void this._upload(blob, mType);
+    };
+
+    // Collect data every 250 ms so we don't lose the tail on stop()
+    this.mediaRecorder.start(250);
+  }
+
+  private _stopRecorder(): void {
+    if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
+      try {
+        this.mediaRecorder.stop();
+      } catch {
+        // ignore
+      }
+    }
+    this.mediaRecorder = null;
+  }
+
+  private _releaseStream(): void {
+    if (this.stream) {
+      this.stream.getTracks().forEach((t) => t.stop());
+      this.stream = null;
+    }
+  }
+
+  private _startVAD(): void {
+    this._stopVAD();
+    this.silenceSince = null;
+    this.isSpeaking = false;
+
+    this.vadTimer = setInterval(() => {
+      if (!this.isActive || !this.analyser) return;
+      const db = this._getRmsDb();
+      const isSilent = db < this.cfg.silenceThresholdDb;
+
+      if (!isSilent) {
+        // Audio detected
+        this.silenceSince = null;
+        if (!this.isSpeaking) {
+          this.isSpeaking = true;
+          // Restart recorder to capture from the start of speech
+          if (this.mediaRecorder?.state === 'recording') {
+            this.recordingStartMs = Date.now();
+          }
+        }
+      } else {
+        // Silence detected
+        if (this.isSpeaking) {
+          if (this.silenceSince === null) {
+            this.silenceSince = Date.now();
+          } else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
+            // End of utterance — stop recorder (triggers onstop → upload)
+            this.isSpeaking = false;
+            this.silenceSince = null;
+            this._finaliseUtterance();
+          }
+        }
+      }
+    }, VAD_POLL_MS);
+  }
+
+  private _stopVAD(): void {
+    if (this.vadTimer !== null) {
+      clearInterval(this.vadTimer);
+      this.vadTimer = null;
+    }
+  }
+
+  /** Stop the current recorder to flush the utterance, then restart it. */
+  private _finaliseUtterance(): void {
+    if (!this.isActive) return;
+    if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
+      this.mediaRecorder.stop();
+    }
+    // Restart recorder for the next utterance after a short delay
+    // (MediaRecorder.onstop fires asynchronously; we wait for it to complete)
+    setTimeout(() => {
+      if (this.isActive && this.stream) {
+        this._startRecorder();
+      }
+    }, 100);
+  }
+
+  /** Compute RMS of current analyser frame in dBFS. */
+  private _getRmsDb(): number {
+    if (!this.analyser) return -Infinity;
+    const buf = new Float32Array(this.analyser.fftSize);
+    this.analyser.getFloatTimeDomainData(buf);
+    let sumSq = 0;
+    for (const s of buf) sumSq += s * s;
+    const rms = Math.sqrt(sumSq / buf.length);
+    return rms === 0 ? -Infinity : 20 * Math.log10(rms);
+  }
+
+  /** POST utterance blob to server, call onResult with transcript. */
+  private async _upload(blob: Blob, mimeType: string): Promise<void> {
+    if (!this.onResult) return;
+
+    try {
+      const headers: Record<string, string> = {
+        'Content-Type': mimeType,
+        'X-Base-URL': this.cfg.baseURL,
+        'X-Model': this.cfg.model,
+      };
+      if (this.cfg.language) {
+        headers['X-Language'] = this.cfg.language;
+      } else if (this.lang && this.lang !== 'auto') {
+        // Use BCP-47 base language code (e.g. 'en' from 'en-US')
+        const baseLang = this.lang.split('-')[0];
+        headers['X-Language'] = baseLang;
+      }
+
+      const response = await fetch('/api/stt/transcribe', {
+        method: 'POST',
+        headers,
+        body: blob,
+      });
+
+      if (!response.ok) {
+        const errData = await response.json().catch(() => ({ error: 'Unknown error' }));
+        throw new Error(errData.error ?? `HTTP ${response.status}`);
+      }
+
+      const data = await response.json();
+      const transcript: string = (data.transcript ?? '').trim();
+      if (transcript) {
+        this.onResult(transcript, true);
+      }
+    } catch (err) {
+      if (!this.isActive) return; // Stopped — ignore
+      const msg = err instanceof Error ? err.message : 'Transcription upload failed';
+      console.error('[AudioStreamService] Upload error:', msg);
+      this.onError?.(msg);
+    }
+  }
+
+  /** Pick the best supported MIME type for MediaRecorder. */
+  private _pickMimeType(): string {
+    const candidates = [
+      'audio/webm;codecs=opus',
+      'audio/webm',
+      'audio/ogg;codecs=opus',
+      'audio/ogg',
+      'audio/mp4',
+    ];
+    if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
+      return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
+    }
+    return '';
+  }
+}
+
+export const audioStreamService = new AudioStreamService();
+export { AudioStreamService };

packages/ui/src/stores/useConfigStore.ts

@@ -477,6 +477,15 @@ interface ConfigStore {
     browserVoice: string;
     openaiVoice: string;
     openaiApiKey: string;
+    openaiCompatibleUrl: string;
+    openaiCompatibleVoice: string;
+    // STT (speech-to-text) settings
+    sttProvider: 'browser' | 'server';
+    sttServerUrl: string;
+    sttModel: string;
+    sttLanguage: string;
+    sttSilenceThresholdDb: number;
+    sttSilenceHoldMs: number;
     showMessageTTSButtons: boolean;
     voiceModeEnabled: boolean;
     // Summarization settings
@@ -491,6 +500,14 @@ interface ConfigStore {
     setBrowserVoice: (voice: string) => void;
     setOpenaiVoice: (voice: string) => void;
     setOpenaiApiKey: (apiKey: string) => void;
+    setOpenaiCompatibleUrl: (url: string) => void;
+    setOpenaiCompatibleVoice: (voice: string) => void;
+    setSttProvider: (provider: 'browser' | 'server') => void;
+    setSttServerUrl: (url: string) => void;
+    setSttModel: (model: string) => void;
+    setSttLanguage: (lang: string) => void;
+    setSttSilenceThresholdDb: (db: number) => void;
+    setSttSilenceHoldMs: (ms: number) => void;
     setShowMessageTTSButtons: (show: boolean) => void;
     setVoiceModeEnabled: (enabled: boolean) => void;
     setSummarizeMessageTTS: (enabled: boolean) => void;
@@ -635,6 +652,71 @@ export const useConfigStore = create<ConfigStore>()(
                     }
                     return '';
                 })(),
+                // OpenAI-compatible custom server URL
+                openaiCompatibleUrl: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('openaiCompatibleUrl');
+                        if (saved) return saved;
+                    }
+                    return '';
+                })(),
+                // OpenAI-compatible custom server voice
+                openaiCompatibleVoice: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('openaiCompatibleVoice');
+                        if (saved) return saved;
+                    }
+                    return 'af_sky';
+                })(),
+                // STT provider: 'browser' (Web Speech API) or 'server' (OpenAI-compat)
+                sttProvider: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('sttProvider');
+                        if (saved === 'browser' || saved === 'server') return saved;
+                    }
+                    return 'browser' as const;
+                })(),
+                sttServerUrl: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('sttServerUrl');
+                        if (saved) return saved;
+                    }
+                    return 'http://localhost:8001/v1';
+                })(),
+                sttModel: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('sttModel');
+                        if (saved) return saved;
+                    }
+                    return 'deepdml/faster-whisper-large-v3-turbo-ct2';
+                })(),
+                sttLanguage: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('sttLanguage');
+                        if (saved !== null) return saved;
+                    }
+                    return '';
+                })(),
+                sttSilenceThresholdDb: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('sttSilenceThresholdDb');
+                        if (saved) {
+                            const parsed = parseFloat(saved);
+                            if (!isNaN(parsed)) return parsed;
+                        }
+                    }
+                    return -45;
+                })(),
+                sttSilenceHoldMs: (() => {
+                    if (typeof window !== 'undefined') {
+                        const saved = localStorage.getItem('sttSilenceHoldMs');
+                        if (saved) {
+                            const parsed = parseInt(saved, 10);
+                            if (!isNaN(parsed)) return parsed;
+                        }
+                    }
+                    return 1500;
+                })(),
                 // Show TTS buttons on messages - disabled by default until user enables it
                 showMessageTTSButtons: (() => {
                     if (typeof window !== 'undefined') {
@@ -1662,6 +1744,62 @@ export const useConfigStore = create<ConfigStore>()(
                     }
                 },
 
+                setOpenaiCompatibleUrl: (url: string) => {
+                    set({ openaiCompatibleUrl: url });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('openaiCompatibleUrl', url);
+                    }
+                },
+
+                setOpenaiCompatibleVoice: (voice: string) => {
+                    set({ openaiCompatibleVoice: voice });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('openaiCompatibleVoice', voice);
+                    }
+                },
+
+                setSttProvider: (provider: 'browser' | 'server') => {
+                    set({ sttProvider: provider });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('sttProvider', provider);
+                    }
+                },
+
+                setSttServerUrl: (url: string) => {
+                    set({ sttServerUrl: url });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('sttServerUrl', url);
+                    }
+                },
+
+                setSttModel: (model: string) => {
+                    set({ sttModel: model });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('sttModel', model);
+                    }
+                },
+
+                setSttLanguage: (lang: string) => {
+                    set({ sttLanguage: lang });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('sttLanguage', lang);
+                    }
+                },
+
+                setSttSilenceThresholdDb: (db: number) => {
+                    set({ sttSilenceThresholdDb: db });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('sttSilenceThresholdDb', String(db));
+                    }
+                },
+
+                setSttSilenceHoldMs: (ms: number) => {
+                    set({ sttSilenceHoldMs: ms });
+                    if (typeof window !== 'undefined') {
+                        localStorage.setItem('sttSilenceHoldMs', String(ms));
+                    }
+                },
+
                 setShowMessageTTSButtons: (show: boolean) => {
                     set({ showMessageTTSButtons: show });
                     if (typeof window !== 'undefined') {

packages/web/server/lib/tts/routes.js

@@ -1,3 +1,5 @@
+import express from 'express';
+
 export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
   let ttsModulePromise = null;
   const getTtsModule = async () => {
@@ -222,4 +224,54 @@ export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
       });
     }
   });
+
+  // Server-side STT: receive raw audio, proxy to OpenAI-compatible transcription endpoint
+  app.post(
+    '/api/stt/transcribe',
+    express.raw({ type: (req) => (req.headers['content-type'] || '').startsWith('audio/'), limit: '20mb' }),
+    async (req, res) => {
+      try {
+        const { transcribeAudio } = await import('./stt.js');
+
+        const mimeType = (req.headers['content-type'] || 'audio/webm').split(',')[0].trim();
+        const baseURL = req.headers['x-base-url'];
+        const model = req.headers['x-model'] || 'deepdml/faster-whisper-large-v3-turbo-ct2';
+        const language = req.headers['x-language'] || undefined;
+
+        if (!req.body || !Buffer.isBuffer(req.body) || req.body.length === 0) {
+          return res.status(400).json({ error: 'Audio data is required' });
+        }
+
+        if (!baseURL) {
+          return res.status(400).json({ error: 'X-Base-URL header is required' });
+        }
+
+        console.log('[STT] Transcribing audio:', {
+          bytes: req.body.length,
+          mimeType,
+          model,
+          baseURL,
+          language,
+        });
+
+        const transcript = await transcribeAudio({
+          audioBuffer: req.body,
+          mimeType,
+          model,
+          baseURL,
+          language,
+        });
+
+        console.log('[STT] Transcript:', transcript?.slice(0, 120));
+        res.json({ transcript: transcript ?? '' });
+      } catch (error) {
+        console.error('[STT] Error:', error);
+        if (!res.headersSent) {
+          res.status(500).json({
+            error: error instanceof Error ? error.message : 'Transcription failed',
+          });
+        }
+      }
+    }
+  );
 }

packages/web/server/lib/tts/stt.js

@@ -0,0 +1,66 @@
+/**
+ * Server-side Speech-to-Text Service
+ *
+ * Proxies audio to any OpenAI-compatible transcription endpoint
+ * (e.g. faster-whisper, whisper.cpp) using the OpenAI Node SDK.
+ */
+
+import OpenAI, { toFile } from 'openai';
+
+/**
+ * Transcribe an audio buffer via an OpenAI-compatible /v1/audio/transcriptions endpoint.
+ *
+ * @param {object} opts
+ * @param {Buffer} opts.audioBuffer  - Raw audio bytes
+ * @param {string} opts.mimeType     - MIME type of the audio (e.g. 'audio/webm')
+ * @param {string} opts.model        - Model name accepted by the remote server
+ * @param {string} [opts.baseURL]    - Base URL of the compatible server (including /v1)
+ * @param {string} [opts.language]   - Optional BCP-47 language hint (e.g. 'en')
+ * @returns {Promise<string>} Transcribed text
+ */
+export async function transcribeAudio({ audioBuffer, mimeType, model, baseURL, language }) {
+  const clientOpts = {
+    apiKey: process.env.OPENAI_API_KEY || 'not-required',
+  };
+  if (baseURL) {
+    clientOpts.baseURL = baseURL;
+  }
+
+  const client = new OpenAI(clientOpts);
+
+  // Derive a sensible filename extension from the MIME type so the server
+  // can infer the codec when it isn't explicit in the stream header.
+  const ext = mimeTypeToExt(mimeType);
+  const filename = `audio.${ext}`;
+
+  const file = await toFile(audioBuffer, filename, { type: mimeType });
+
+  const result = await client.audio.transcriptions.create({
+    file,
+    model,
+    response_format: 'json',
+    ...(language ? { language } : {}),
+  });
+
+  return result.text ?? '';
+}
+
+/**
+ * Map a MIME type to a file extension understood by Whisper servers.
+ * @param {string} mimeType
+ * @returns {string}
+ */
+function mimeTypeToExt(mimeType) {
+  const type = (mimeType || '').split(';')[0].trim().toLowerCase();
+  const map = {
+    'audio/webm': 'webm',
+    'audio/ogg': 'ogg',
+    'audio/wav': 'wav',
+    'audio/wave': 'wav',
+    'audio/mpeg': 'mp3',
+    'audio/mp4': 'mp4',
+    'audio/mp3': 'mp3',
+    'audio/flac': 'flac',
+  };
+  return map[type] ?? 'webm';
+}