feat(stt): add server-side STT provider via OpenAI-compatible Whisper endpoint (#860)
Co-authored-by: Alexander Busse <alex@ableph.net>
This commit is contained in:
Alexander Busse
committed by
GitHub
GitHub
co-authored by
Alexander Busse
Alexander Busse
parent
75a10ea66c
commit
3746d99a85
@@ -16,6 +16,7 @@ import { NumberInput } from '@/components/ui/number-input';
|
||||
import { RiPlayLine, RiStopLine, RiCloseLine, RiAppleLine, RiInformationLine } from '@remixicon/react';
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { audioStreamService } from '@/lib/voice/audioStreamService';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
const LANGUAGE_OPTIONS = [
|
||||
@@ -71,6 +72,19 @@ export const VoiceSettings: React.FC = () => {
|
||||
const openaiApiKey = useConfigStore((state) => state.openaiApiKey);
|
||||
const setOpenaiApiKey = useConfigStore((state) => state.setOpenaiApiKey);
|
||||
const showMessageTTSButtons = useConfigStore((state) => state.showMessageTTSButtons);
|
||||
// STT settings
|
||||
const sttProvider = useConfigStore((state) => state.sttProvider);
|
||||
const setSttProvider = useConfigStore((state) => state.setSttProvider);
|
||||
const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
|
||||
const setSttServerUrl = useConfigStore((state) => state.setSttServerUrl);
|
||||
const sttModel = useConfigStore((state) => state.sttModel);
|
||||
const setSttModel = useConfigStore((state) => state.setSttModel);
|
||||
const sttLanguage = useConfigStore((state) => state.sttLanguage);
|
||||
const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
|
||||
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
|
||||
const setSttSilenceThresholdDb = useConfigStore((state) => state.setSttSilenceThresholdDb);
|
||||
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
|
||||
const setSttSilenceHoldMs = useConfigStore((state) => state.setSttSilenceHoldMs);
|
||||
const setShowMessageTTSButtons = useConfigStore((state) => state.setShowMessageTTSButtons);
|
||||
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
|
||||
const setVoiceModeEnabled = useConfigStore((state) => state.setVoiceModeEnabled);
|
||||
@@ -563,6 +577,146 @@ export const VoiceSettings: React.FC = () => {
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{/* Speech Recognition */}
|
||||
{voiceModeEnabled && (
|
||||
<div className="mb-8">
|
||||
<div className="mb-1 px-1">
|
||||
<h3 className="typography-ui-header font-medium text-foreground">
|
||||
Speech Recognition
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
<section className="px-2 pb-2 pt-0 space-y-0">
|
||||
<div className="pb-1.5 pt-0.5">
|
||||
<div className="flex min-w-0 flex-col gap-1.5">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="typography-ui-label text-foreground">Provider</span>
|
||||
<Tooltip delayDuration={1000}>
|
||||
<TooltipTrigger asChild>
|
||||
<RiInformationLine className="h-3.5 w-3.5 text-muted-foreground/60 cursor-help" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent sideOffset={8} className="max-w-xs">
|
||||
<ul className="space-y-1">
|
||||
<li><strong>Browser:</strong> Web Speech API (Chrome/Edge). Free, no setup.</li>
|
||||
<li><strong>Server:</strong> OpenAI-compatible Whisper server. Better accuracy, any language.</li>
|
||||
</ul>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="xs"
|
||||
onClick={() => setSttProvider('browser')}
|
||||
className={cn(
|
||||
'!font-normal',
|
||||
sttProvider === 'browser'
|
||||
? 'border-[var(--primary-base)] text-[var(--primary-base)] bg-[var(--primary-base)]/10 hover:text-[var(--primary-base)]'
|
||||
: 'text-foreground'
|
||||
)}
|
||||
>
|
||||
Browser
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="xs"
|
||||
onClick={() => setSttProvider('server')}
|
||||
className={cn(
|
||||
'!font-normal',
|
||||
sttProvider === 'server'
|
||||
? 'border-[var(--primary-base)] text-[var(--primary-base)] bg-[var(--primary-base)]/10 hover:text-[var(--primary-base)]'
|
||||
: 'text-foreground'
|
||||
)}
|
||||
>
|
||||
Server
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{sttProvider === 'server' && (
|
||||
<div className="py-1.5 space-y-2">
|
||||
{!audioStreamService.isSupported() && (
|
||||
<p className="typography-meta text-[var(--status-error)]">
|
||||
MediaRecorder or AudioContext is not available in this browser. Server STT may not work.
|
||||
</p>
|
||||
)}
|
||||
<div>
|
||||
<span className={cn("typography-ui-label text-foreground", !sttServerUrl.trim() && "text-[var(--status-error)]")}>
|
||||
Server URL
|
||||
</span>
|
||||
<span className="typography-meta ml-2 text-muted-foreground">
|
||||
Base URL of the Whisper-compatible server
|
||||
</span>
|
||||
<div className="relative mt-1.5 max-w-xs">
|
||||
<input
|
||||
type="text"
|
||||
value={sttServerUrl}
|
||||
onChange={(e) => setSttServerUrl(e.target.value)}
|
||||
placeholder="http://localhost:8001/v1"
|
||||
className="w-full h-7 rounded-lg border border-input bg-transparent px-2 typography-ui-label text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-primary/50 focus:border-primary/70"
|
||||
/>
|
||||
{sttServerUrl && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSttServerUrl('')}
|
||||
className="absolute right-2 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<RiCloseLine className="w-3.5 h-3.5" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<span className="typography-ui-label text-foreground">Model</span>
|
||||
<div className="relative mt-1.5 max-w-xs">
|
||||
<input
|
||||
type="text"
|
||||
value={sttModel}
|
||||
onChange={(e) => setSttModel(e.target.value)}
|
||||
placeholder="deepdml/faster-whisper-large-v3-turbo-ct2"
|
||||
className="w-full h-7 rounded-lg border border-input bg-transparent px-2 typography-ui-label text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-primary/50 focus:border-primary/70"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<span className="typography-ui-label text-foreground">Language</span>
|
||||
<span className="typography-meta ml-2 text-muted-foreground">
|
||||
BCP-47 code (e.g. en, fr). Leave blank for auto-detect.
|
||||
</span>
|
||||
<div className="relative mt-1.5 max-w-[8rem]">
|
||||
<input
|
||||
type="text"
|
||||
value={sttLanguage}
|
||||
onChange={(e) => setSttLanguage(e.target.value)}
|
||||
placeholder="auto"
|
||||
className="w-full h-7 rounded-lg border border-input bg-transparent px-2 typography-ui-label text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-primary/50 focus:border-primary/70"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-8 py-0.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">Silence Threshold</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={-60} max={-20} step={1} value={sttSilenceThresholdDb} onChange={(e) => setSttSilenceThresholdDb(Number(e.target.value))} className={sliderClass} />}
|
||||
<span className="typography-ui-label text-foreground tabular-nums min-w-[3.5rem] text-right">
|
||||
{sttSilenceThresholdDb} dB
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-8 py-0.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">Silence Hold</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={500} max={3000} step={100} value={sttSilenceHoldMs} onChange={(e) => setSttSilenceHoldMs(Number(e.target.value))} className={sliderClass} />}
|
||||
<NumberInput value={sttSilenceHoldMs} onValueChange={setSttSilenceHoldMs} min={500} max={3000} step={100} className="w-20 tabular-nums" />
|
||||
<span className="typography-meta text-muted-foreground">ms</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Playback & Summarization */}
|
||||
<div className="mb-8">
|
||||
<div className="mb-1 px-1">
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { audioStreamService } from '@/lib/voice/audioStreamService';
|
||||
import { useSessionUIStore } from '@/sync/session-ui-store';
|
||||
import { useInputStore } from '@/sync/input-store';
|
||||
import { getSyncMessages, getSyncParts } from '@/sync/sync-refs';
|
||||
@@ -109,8 +110,6 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
return false;
|
||||
});
|
||||
|
||||
const isSupported = browserVoiceService.isSupported();
|
||||
|
||||
// Mobile detection
|
||||
const isMobile = useMemo(() => {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
@@ -143,12 +142,26 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
|
||||
const openaiCompatibleUrl = useConfigStore((state) => state.openaiCompatibleUrl);
|
||||
const summarizeVoiceConversation = useConfigStore((state) => state.summarizeVoiceConversation);
|
||||
const summarizeCharacterThreshold = useConfigStore((state) => state.summarizeCharacterThreshold);
|
||||
|
||||
const shouldCheckOpenAIAvailability = voiceModeEnabled && voiceProvider === 'openai';
|
||||
const shouldCheckSayAvailability = voiceModeEnabled && voiceProvider === 'say';
|
||||
|
||||
// STT provider config
|
||||
const sttProvider = useConfigStore((state) => state.sttProvider);
|
||||
const sttServerUrl = useConfigStore((state) => state.sttServerUrl);
|
||||
const sttModel = useConfigStore((state) => state.sttModel);
|
||||
const sttLanguage = useConfigStore((state) => state.sttLanguage);
|
||||
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
|
||||
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
|
||||
|
||||
const isSupported = sttProvider === 'server'
|
||||
? audioStreamService.isSupported()
|
||||
: browserVoiceService.isSupported();
|
||||
|
||||
// Server TTS for mobile (bypasses Safari audio restrictions)
|
||||
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS({
|
||||
enabled: shouldCheckOpenAIAvailability,
|
||||
@@ -170,6 +183,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
isActiveRef.current = false;
|
||||
processingMessageRef.current = false;
|
||||
browserVoiceService.stopListening();
|
||||
audioStreamService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
setStatus('idle');
|
||||
setError(null);
|
||||
@@ -309,11 +323,15 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
if (isActiveRef.current) {
|
||||
setStatus('listening');
|
||||
setError(null);
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
|
||||
if (sttProvider === 'server') {
|
||||
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechError).catch(() => {});
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
}, [language, conversationMode]);
|
||||
}, [language, conversationMode, sttProvider]);
|
||||
|
||||
// Update the ref when handleSpeechError changes
|
||||
useEffect(() => {
|
||||
@@ -336,6 +354,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
|
||||
// Stop listening while processing
|
||||
browserVoiceService.stopListening();
|
||||
audioStreamService.stopListening();
|
||||
|
||||
// Non-continuous mode: fill chat input only, do not auto-send.
|
||||
if (!conversationMode) {
|
||||
@@ -423,7 +442,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
}
|
||||
|
||||
setStatus('listening');
|
||||
if (isMobile) {
|
||||
if (sttProvider === 'server') {
|
||||
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
|
||||
console.error('[useBrowserVoice] Failed to restart server STT:', err);
|
||||
});
|
||||
} else if (isMobile) {
|
||||
try {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} catch (err) {
|
||||
@@ -514,7 +537,11 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
// Only restart listening if conversation mode is enabled
|
||||
if (conversationMode) {
|
||||
setStatus('listening');
|
||||
if (isMobile) {
|
||||
if (sttProvider === 'server') {
|
||||
audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => {
|
||||
console.error('[useBrowserVoice] Failed to restart server STT after speech error:', err);
|
||||
});
|
||||
} else if (isMobile) {
|
||||
try {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} catch (restartErr) {
|
||||
@@ -546,7 +573,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
setStatus('error');
|
||||
processingMessageRef.current = false;
|
||||
}
|
||||
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode]);
|
||||
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, sttProvider]);
|
||||
|
||||
// Handle speech recognition result
|
||||
const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
|
||||
@@ -589,7 +616,9 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
setStatus('listening');
|
||||
try {
|
||||
if (isMobile) {
|
||||
if (sttProvider === 'server') {
|
||||
void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
@@ -605,7 +634,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
return () => {
|
||||
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
||||
};
|
||||
}, [conversationMode, isMobile, language]);
|
||||
}, [conversationMode, isMobile, language, sttProvider]);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof navigator === 'undefined') {
|
||||
@@ -640,11 +669,16 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
}
|
||||
|
||||
try {
|
||||
browserVoiceService.stopListening();
|
||||
if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
if (sttProvider === 'server') {
|
||||
audioStreamService.stopListening();
|
||||
void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
browserVoiceService.stopListening();
|
||||
if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
|
||||
@@ -664,7 +698,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [isMobile, language, status]);
|
||||
}, [isMobile, language, status, sttProvider]);
|
||||
|
||||
// Update the ref when handleSpeechResult changes
|
||||
useEffect(() => {
|
||||
@@ -676,6 +710,10 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
if (!isSupported) {
|
||||
return false;
|
||||
}
|
||||
if (sttProvider === 'server') {
|
||||
// getUserMedia permission is requested on startListening; nothing to prepare
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
await browserVoiceService.prepareListening();
|
||||
return true;
|
||||
@@ -684,12 +722,12 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
setError(errorMsg);
|
||||
return false;
|
||||
}
|
||||
}, [isSupported]);
|
||||
}, [isSupported, sttProvider]);
|
||||
|
||||
// Start voice mode
|
||||
const startVoice = useCallback(async () => {
|
||||
if (!isSupported) {
|
||||
setError('Browser voice not supported');
|
||||
setError('Voice input not supported in this browser');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
@@ -704,7 +742,29 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
lastTranscriptRef.current = '';
|
||||
setError(null);
|
||||
setStatus('listening');
|
||||
|
||||
|
||||
if (sttProvider === 'server') {
|
||||
// Server STT: configure the service then start async recording
|
||||
audioStreamService.configure({
|
||||
baseURL: sttServerUrl,
|
||||
model: sttModel,
|
||||
language: sttLanguage || undefined,
|
||||
silenceThresholdDb: sttSilenceThresholdDb,
|
||||
silenceHoldMs: sttSilenceHoldMs,
|
||||
});
|
||||
try {
|
||||
await audioStreamService.startListening(language, handleSpeechResult, handleSpeechError);
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
|
||||
console.error('[useBrowserVoice] Server STT start error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Browser STT
|
||||
// On mobile, use sync path to ensure SpeechRecognition.start() is called
|
||||
// within the same user gesture context (required by iOS Safari)
|
||||
// Also unlock audio immediately for TTS playback later
|
||||
@@ -742,7 +802,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
}
|
||||
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio]);
|
||||
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]);
|
||||
|
||||
// Stop voice mode
|
||||
const stopVoice = useCallback(() => {
|
||||
@@ -759,6 +819,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
finalTranscriptTimerRef.current = null;
|
||||
}
|
||||
browserVoiceService.stopListening();
|
||||
audioStreamService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
stopServerTTS(); // Also stop server TTS if playing
|
||||
stopSayTTS(); // Also stop Say TTS if playing
|
||||
@@ -781,6 +842,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
}
|
||||
browserVoiceService.setConversationMode(false);
|
||||
browserVoiceService.stopListening();
|
||||
audioStreamService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
/**
|
||||
* Audio Stream Service
|
||||
*
|
||||
* Captures microphone audio using MediaRecorder, detects utterance boundaries
|
||||
* via an AnalyserNode-based silence detector (VAD), then POSTs each utterance
|
||||
* as a raw audio blob to the OpenChamber server's /api/stt/transcribe endpoint.
|
||||
*
|
||||
* Mimics the BrowserVoiceService.startListening interface so useBrowserVoice
|
||||
* can swap providers without changing its internal logic.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* audioStreamService.configure({ baseURL: 'http://localhost:8001/v1', model: 'whisper-1' });
|
||||
* audioStreamService.startListening('en', (text, isFinal) => {
|
||||
* if (isFinal) console.log('transcript:', text);
|
||||
* });
|
||||
* audioStreamService.stopListening();
|
||||
* ```
|
||||
*/
|
||||
|
||||
export type SpeechResultCallback = (text: string, isFinal: boolean) => void;
|
||||
export type ErrorCallback = (error: string) => void;
|
||||
|
||||
export interface AudioStreamConfig {
|
||||
/** Base URL of the OpenAI-compatible STT server (e.g. http://localhost:8001/v1) */
|
||||
baseURL: string;
|
||||
/** Whisper-compatible model name */
|
||||
model: string;
|
||||
/** Optional BCP-47 language hint (e.g. 'en'). Empty string = auto-detect. */
|
||||
language?: string;
|
||||
/**
|
||||
* Silence threshold in dB below which audio is considered silence.
|
||||
* Lower (more negative) = only very quiet audio counts as silence.
|
||||
* Default: -45
|
||||
*/
|
||||
silenceThresholdDb?: number;
|
||||
/**
|
||||
* How long continuous silence must last (ms) before the utterance is finalised.
|
||||
* Default: 1500
|
||||
*/
|
||||
silenceHoldMs?: number;
|
||||
}
|
||||
|
||||
// How often (ms) the VAD samples the analyser
|
||||
const VAD_POLL_MS = 80;
|
||||
// Minimum audio duration (ms) to bother uploading (avoids blank clips)
|
||||
const MIN_UTTERANCE_MS = 300;
|
||||
|
||||
class AudioStreamService {
|
||||
private stream: MediaStream | null = null;
|
||||
private mediaRecorder: MediaRecorder | null = null;
|
||||
private audioContext: AudioContext | null = null;
|
||||
private analyser: AnalyserNode | null = null;
|
||||
private vadTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private chunks: Blob[] = [];
|
||||
private recordingStartMs = 0;
|
||||
private isActive = false;
|
||||
private isSpeaking = false;
|
||||
private silenceSince: number | null = null;
|
||||
private onResult: SpeechResultCallback | null = null;
|
||||
private onError: ErrorCallback | null = null;
|
||||
private lang = 'en';
|
||||
|
||||
// Configurable parameters
|
||||
private cfg: Required<AudioStreamConfig> = {
|
||||
baseURL: '',
|
||||
model: 'deepdml/faster-whisper-large-v3-turbo-ct2',
|
||||
language: '',
|
||||
silenceThresholdDb: -45,
|
||||
silenceHoldMs: 1500,
|
||||
};
|
||||
|
||||
/** Update service configuration. Can be called before or after startListening. */
|
||||
configure(config: AudioStreamConfig): void {
|
||||
this.cfg = {
|
||||
silenceThresholdDb: -45,
|
||||
silenceHoldMs: 1500,
|
||||
language: '',
|
||||
...config,
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether the browser supports the required APIs. */
|
||||
isSupported(): boolean {
|
||||
return (
|
||||
typeof window !== 'undefined' &&
|
||||
typeof navigator !== 'undefined' &&
|
||||
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
|
||||
typeof window.MediaRecorder !== 'undefined' &&
|
||||
typeof window.AudioContext !== 'undefined'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start listening. Requests microphone access if not already held.
|
||||
* Calls onResult(text, true) for each completed utterance.
|
||||
*/
|
||||
async startListening(
|
||||
lang: string,
|
||||
onResult: SpeechResultCallback,
|
||||
onError?: ErrorCallback
|
||||
): Promise<void> {
|
||||
if (this.isActive) {
|
||||
this.stopListening();
|
||||
}
|
||||
|
||||
this.lang = lang;
|
||||
this.onResult = onResult;
|
||||
this.onError = onError ?? null;
|
||||
this.isActive = true;
|
||||
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
||||
} catch (err) {
|
||||
this.isActive = false;
|
||||
const msg = err instanceof Error ? err.message : 'Microphone access denied';
|
||||
onError?.(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
this._setupAudioContext();
|
||||
this._startRecorder();
|
||||
this._startVAD();
|
||||
}
|
||||
|
||||
/** Stop listening and clean up all resources. */
|
||||
stopListening(): void {
|
||||
this.isActive = false;
|
||||
this._stopVAD();
|
||||
this._stopRecorder();
|
||||
this._teardownAudioContext();
|
||||
this._releaseStream();
|
||||
this.chunks = [];
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
this.onResult = null;
|
||||
this.onError = null;
|
||||
}
|
||||
|
||||
/** Whether currently listening. */
|
||||
getIsListening(): boolean {
|
||||
return this.isActive;
|
||||
}
|
||||
|
||||
// ── Private helpers ──────────────────────────────────────────────────────
|
||||
|
||||
private _setupAudioContext(): void {
|
||||
if (!this.stream) return;
|
||||
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
|
||||
this.audioContext = new AudioContextClass();
|
||||
const source = this.audioContext.createMediaStreamSource(this.stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 512;
|
||||
source.connect(this.analyser);
|
||||
}
|
||||
|
||||
private _teardownAudioContext(): void {
|
||||
try {
|
||||
this.audioContext?.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
}
|
||||
|
||||
private _startRecorder(): void {
|
||||
if (!this.stream) return;
|
||||
|
||||
const mimeType = this._pickMimeType();
|
||||
const options: MediaRecorderOptions = {};
|
||||
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
|
||||
options.mimeType = mimeType;
|
||||
}
|
||||
|
||||
this.mediaRecorder = new MediaRecorder(this.stream, options);
|
||||
this.chunks = [];
|
||||
this.recordingStartMs = Date.now();
|
||||
|
||||
this.mediaRecorder.ondataavailable = (e) => {
|
||||
if (e.data && e.data.size > 0) {
|
||||
this.chunks.push(e.data);
|
||||
}
|
||||
};
|
||||
|
||||
this.mediaRecorder.onstop = () => {
|
||||
const blobs = this.chunks.splice(0);
|
||||
const durationMs = Date.now() - this.recordingStartMs;
|
||||
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) return;
|
||||
|
||||
const mType = blobs[0].type || mimeType || 'audio/webm';
|
||||
const blob = new Blob(blobs, { type: mType });
|
||||
void this._upload(blob, mType);
|
||||
};
|
||||
|
||||
// Collect data every 250 ms so we don't lose the tail on stop()
|
||||
this.mediaRecorder.start(250);
|
||||
}
|
||||
|
||||
private _stopRecorder(): void {
|
||||
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
|
||||
try {
|
||||
this.mediaRecorder.stop();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
this.mediaRecorder = null;
|
||||
}
|
||||
|
||||
private _releaseStream(): void {
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
}
|
||||
|
||||
private _startVAD(): void {
|
||||
this._stopVAD();
|
||||
this.silenceSince = null;
|
||||
this.isSpeaking = false;
|
||||
|
||||
this.vadTimer = setInterval(() => {
|
||||
if (!this.isActive || !this.analyser) return;
|
||||
const db = this._getRmsDb();
|
||||
const isSilent = db < this.cfg.silenceThresholdDb;
|
||||
|
||||
if (!isSilent) {
|
||||
// Audio detected
|
||||
this.silenceSince = null;
|
||||
if (!this.isSpeaking) {
|
||||
this.isSpeaking = true;
|
||||
// Restart recorder to capture from the start of speech
|
||||
if (this.mediaRecorder?.state === 'recording') {
|
||||
this.recordingStartMs = Date.now();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Silence detected
|
||||
if (this.isSpeaking) {
|
||||
if (this.silenceSince === null) {
|
||||
this.silenceSince = Date.now();
|
||||
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
|
||||
// End of utterance — stop recorder (triggers onstop → upload)
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
this._finaliseUtterance();
|
||||
}
|
||||
}
|
||||
}
|
||||
}, VAD_POLL_MS);
|
||||
}
|
||||
|
||||
private _stopVAD(): void {
|
||||
if (this.vadTimer !== null) {
|
||||
clearInterval(this.vadTimer);
|
||||
this.vadTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Stop the current recorder to flush the utterance, then restart it. */
|
||||
private _finaliseUtterance(): void {
|
||||
if (!this.isActive) return;
|
||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
||||
this.mediaRecorder.stop();
|
||||
}
|
||||
// Restart recorder for the next utterance after a short delay
|
||||
// (MediaRecorder.onstop fires asynchronously; we wait for it to complete)
|
||||
setTimeout(() => {
|
||||
if (this.isActive && this.stream) {
|
||||
this._startRecorder();
|
||||
}
|
||||
}, 100);
|
||||
}
|
||||
|
||||
/** Compute RMS of current analyser frame in dBFS. */
|
||||
private _getRmsDb(): number {
|
||||
if (!this.analyser) return -Infinity;
|
||||
const buf = new Float32Array(this.analyser.fftSize);
|
||||
this.analyser.getFloatTimeDomainData(buf);
|
||||
let sumSq = 0;
|
||||
for (const s of buf) sumSq += s * s;
|
||||
const rms = Math.sqrt(sumSq / buf.length);
|
||||
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
|
||||
}
|
||||
|
||||
/** POST utterance blob to server, call onResult with transcript. */
|
||||
private async _upload(blob: Blob, mimeType: string): Promise<void> {
|
||||
if (!this.onResult) return;
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': mimeType,
|
||||
'X-Base-URL': this.cfg.baseURL,
|
||||
'X-Model': this.cfg.model,
|
||||
};
|
||||
if (this.cfg.language) {
|
||||
headers['X-Language'] = this.cfg.language;
|
||||
} else if (this.lang && this.lang !== 'auto') {
|
||||
// Use BCP-47 base language code (e.g. 'en' from 'en-US')
|
||||
const baseLang = this.lang.split('-')[0];
|
||||
headers['X-Language'] = baseLang;
|
||||
}
|
||||
|
||||
const response = await fetch('/api/stt/transcribe', {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: blob,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errData.error ?? `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const transcript: string = (data.transcript ?? '').trim();
|
||||
if (transcript) {
|
||||
this.onResult(transcript, true);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!this.isActive) return; // Stopped — ignore
|
||||
const msg = err instanceof Error ? err.message : 'Transcription upload failed';
|
||||
console.error('[AudioStreamService] Upload error:', msg);
|
||||
this.onError?.(msg);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pick the best supported MIME type for MediaRecorder. */
|
||||
private _pickMimeType(): string {
|
||||
const candidates = [
|
||||
'audio/webm;codecs=opus',
|
||||
'audio/webm',
|
||||
'audio/ogg;codecs=opus',
|
||||
'audio/ogg',
|
||||
'audio/mp4',
|
||||
];
|
||||
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
|
||||
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
export const audioStreamService = new AudioStreamService();
|
||||
export { AudioStreamService };
|
||||
@@ -477,6 +477,15 @@ interface ConfigStore {
|
||||
browserVoice: string;
|
||||
openaiVoice: string;
|
||||
openaiApiKey: string;
|
||||
openaiCompatibleUrl: string;
|
||||
openaiCompatibleVoice: string;
|
||||
// STT (speech-to-text) settings
|
||||
sttProvider: 'browser' | 'server';
|
||||
sttServerUrl: string;
|
||||
sttModel: string;
|
||||
sttLanguage: string;
|
||||
sttSilenceThresholdDb: number;
|
||||
sttSilenceHoldMs: number;
|
||||
showMessageTTSButtons: boolean;
|
||||
voiceModeEnabled: boolean;
|
||||
// Summarization settings
|
||||
@@ -491,6 +500,14 @@ interface ConfigStore {
|
||||
setBrowserVoice: (voice: string) => void;
|
||||
setOpenaiVoice: (voice: string) => void;
|
||||
setOpenaiApiKey: (apiKey: string) => void;
|
||||
setOpenaiCompatibleUrl: (url: string) => void;
|
||||
setOpenaiCompatibleVoice: (voice: string) => void;
|
||||
setSttProvider: (provider: 'browser' | 'server') => void;
|
||||
setSttServerUrl: (url: string) => void;
|
||||
setSttModel: (model: string) => void;
|
||||
setSttLanguage: (lang: string) => void;
|
||||
setSttSilenceThresholdDb: (db: number) => void;
|
||||
setSttSilenceHoldMs: (ms: number) => void;
|
||||
setShowMessageTTSButtons: (show: boolean) => void;
|
||||
setVoiceModeEnabled: (enabled: boolean) => void;
|
||||
setSummarizeMessageTTS: (enabled: boolean) => void;
|
||||
@@ -635,6 +652,71 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
// OpenAI-compatible custom server URL
|
||||
openaiCompatibleUrl: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('openaiCompatibleUrl');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
// OpenAI-compatible custom server voice
|
||||
openaiCompatibleVoice: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('openaiCompatibleVoice');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return 'af_sky';
|
||||
})(),
|
||||
// STT provider: 'browser' (Web Speech API) or 'server' (OpenAI-compat)
|
||||
sttProvider: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttProvider');
|
||||
if (saved === 'browser' || saved === 'server') return saved;
|
||||
}
|
||||
return 'browser' as const;
|
||||
})(),
|
||||
sttServerUrl: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttServerUrl');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return 'http://localhost:8001/v1';
|
||||
})(),
|
||||
sttModel: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttModel');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return 'deepdml/faster-whisper-large-v3-turbo-ct2';
|
||||
})(),
|
||||
sttLanguage: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttLanguage');
|
||||
if (saved !== null) return saved;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
sttSilenceThresholdDb: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttSilenceThresholdDb');
|
||||
if (saved) {
|
||||
const parsed = parseFloat(saved);
|
||||
if (!isNaN(parsed)) return parsed;
|
||||
}
|
||||
}
|
||||
return -45;
|
||||
})(),
|
||||
sttSilenceHoldMs: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttSilenceHoldMs');
|
||||
if (saved) {
|
||||
const parsed = parseInt(saved, 10);
|
||||
if (!isNaN(parsed)) return parsed;
|
||||
}
|
||||
}
|
||||
return 1500;
|
||||
})(),
|
||||
// Show TTS buttons on messages - disabled by default until user enables it
|
||||
showMessageTTSButtons: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -1662,6 +1744,62 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
},
|
||||
|
||||
setOpenaiCompatibleUrl: (url: string) => {
|
||||
set({ openaiCompatibleUrl: url });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('openaiCompatibleUrl', url);
|
||||
}
|
||||
},
|
||||
|
||||
setOpenaiCompatibleVoice: (voice: string) => {
|
||||
set({ openaiCompatibleVoice: voice });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('openaiCompatibleVoice', voice);
|
||||
}
|
||||
},
|
||||
|
||||
setSttProvider: (provider: 'browser' | 'server') => {
|
||||
set({ sttProvider: provider });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttProvider', provider);
|
||||
}
|
||||
},
|
||||
|
||||
setSttServerUrl: (url: string) => {
|
||||
set({ sttServerUrl: url });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttServerUrl', url);
|
||||
}
|
||||
},
|
||||
|
||||
setSttModel: (model: string) => {
|
||||
set({ sttModel: model });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttModel', model);
|
||||
}
|
||||
},
|
||||
|
||||
setSttLanguage: (lang: string) => {
|
||||
set({ sttLanguage: lang });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttLanguage', lang);
|
||||
}
|
||||
},
|
||||
|
||||
setSttSilenceThresholdDb: (db: number) => {
|
||||
set({ sttSilenceThresholdDb: db });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttSilenceThresholdDb', String(db));
|
||||
}
|
||||
},
|
||||
|
||||
setSttSilenceHoldMs: (ms: number) => {
|
||||
set({ sttSilenceHoldMs: ms });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttSilenceHoldMs', String(ms));
|
||||
}
|
||||
},
|
||||
|
||||
setShowMessageTTSButtons: (show: boolean) => {
|
||||
set({ showMessageTTSButtons: show });
|
||||
if (typeof window !== 'undefined') {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import express from 'express';
|
||||
|
||||
export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
|
||||
let ttsModulePromise = null;
|
||||
const getTtsModule = async () => {
|
||||
@@ -222,4 +224,54 @@ export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Server-side STT: receive raw audio, proxy to OpenAI-compatible transcription endpoint
|
||||
app.post(
|
||||
'/api/stt/transcribe',
|
||||
express.raw({ type: (req) => (req.headers['content-type'] || '').startsWith('audio/'), limit: '20mb' }),
|
||||
async (req, res) => {
|
||||
try {
|
||||
const { transcribeAudio } = await import('./stt.js');
|
||||
|
||||
const mimeType = (req.headers['content-type'] || 'audio/webm').split(',')[0].trim();
|
||||
const baseURL = req.headers['x-base-url'];
|
||||
const model = req.headers['x-model'] || 'deepdml/faster-whisper-large-v3-turbo-ct2';
|
||||
const language = req.headers['x-language'] || undefined;
|
||||
|
||||
if (!req.body || !Buffer.isBuffer(req.body) || req.body.length === 0) {
|
||||
return res.status(400).json({ error: 'Audio data is required' });
|
||||
}
|
||||
|
||||
if (!baseURL) {
|
||||
return res.status(400).json({ error: 'X-Base-URL header is required' });
|
||||
}
|
||||
|
||||
console.log('[STT] Transcribing audio:', {
|
||||
bytes: req.body.length,
|
||||
mimeType,
|
||||
model,
|
||||
baseURL,
|
||||
language,
|
||||
});
|
||||
|
||||
const transcript = await transcribeAudio({
|
||||
audioBuffer: req.body,
|
||||
mimeType,
|
||||
model,
|
||||
baseURL,
|
||||
language,
|
||||
});
|
||||
|
||||
console.log('[STT] Transcript:', transcript?.slice(0, 120));
|
||||
res.json({ transcript: transcript ?? '' });
|
||||
} catch (error) {
|
||||
console.error('[STT] Error:', error);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({
|
||||
error: error instanceof Error ? error.message : 'Transcription failed',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Server-side Speech-to-Text Service
|
||||
*
|
||||
* Proxies audio to any OpenAI-compatible transcription endpoint
|
||||
* (e.g. faster-whisper, whisper.cpp) using the OpenAI Node SDK.
|
||||
*/
|
||||
|
||||
import OpenAI, { toFile } from 'openai';
|
||||
|
||||
/**
|
||||
* Transcribe an audio buffer via an OpenAI-compatible /v1/audio/transcriptions endpoint.
|
||||
*
|
||||
* @param {object} opts
|
||||
* @param {Buffer} opts.audioBuffer - Raw audio bytes
|
||||
* @param {string} opts.mimeType - MIME type of the audio (e.g. 'audio/webm')
|
||||
* @param {string} opts.model - Model name accepted by the remote server
|
||||
* @param {string} [opts.baseURL] - Base URL of the compatible server (including /v1)
|
||||
* @param {string} [opts.language] - Optional BCP-47 language hint (e.g. 'en')
|
||||
* @returns {Promise<string>} Transcribed text
|
||||
*/
|
||||
export async function transcribeAudio({ audioBuffer, mimeType, model, baseURL, language }) {
|
||||
const clientOpts = {
|
||||
apiKey: process.env.OPENAI_API_KEY || 'not-required',
|
||||
};
|
||||
if (baseURL) {
|
||||
clientOpts.baseURL = baseURL;
|
||||
}
|
||||
|
||||
const client = new OpenAI(clientOpts);
|
||||
|
||||
// Derive a sensible filename extension from the MIME type so the server
|
||||
// can infer the codec when it isn't explicit in the stream header.
|
||||
const ext = mimeTypeToExt(mimeType);
|
||||
const filename = `audio.${ext}`;
|
||||
|
||||
const file = await toFile(audioBuffer, filename, { type: mimeType });
|
||||
|
||||
const result = await client.audio.transcriptions.create({
|
||||
file,
|
||||
model,
|
||||
response_format: 'json',
|
||||
...(language ? { language } : {}),
|
||||
});
|
||||
|
||||
return result.text ?? '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a MIME type to a file extension understood by Whisper servers.
|
||||
* @param {string} mimeType
|
||||
* @returns {string}
|
||||
*/
|
||||
function mimeTypeToExt(mimeType) {
|
||||
const type = (mimeType || '').split(';')[0].trim().toLowerCase();
|
||||
const map = {
|
||||
'audio/webm': 'webm',
|
||||
'audio/ogg': 'ogg',
|
||||
'audio/wav': 'wav',
|
||||
'audio/wave': 'wav',
|
||||
'audio/mpeg': 'mp3',
|
||||
'audio/mp4': 'mp4',
|
||||
'audio/mp3': 'mp3',
|
||||
'audio/flac': 'flac',
|
||||
};
|
||||
return map[type] ?? 'webm';
|
||||
}
|
||||
Reference in New Issue
Block a user