feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)
Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
This commit is contained in:
committed by
GitHub
parent
3f5151d424
commit
de1b85ac56
@@ -44,7 +44,6 @@ import { ConfigUpdateOverlay } from '@/components/ui/ConfigUpdateOverlay';
|
||||
import { AboutDialog } from '@/components/ui/AboutDialog';
|
||||
import { RuntimeAPIProvider } from '@/contexts/RuntimeAPIProvider';
|
||||
import { registerRuntimeAPIs } from '@/contexts/runtimeAPIRegistry';
|
||||
import { VoiceProvider } from '@/components/voice';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
import { useGitHubAuthStore } from '@/stores/useGitHubAuthStore';
|
||||
import { useFeatureFlagsStore } from '@/stores/useFeatureFlagsStore';
|
||||
@@ -928,8 +927,8 @@ function App({ apis }: AppProps) {
|
||||
}
|
||||
|
||||
// Always mount the full provider tree to avoid remounts when isInitialized
|
||||
// flips from false → true. FireworksProvider and VoiceProvider are lightweight
|
||||
// shells; their heavy children are only activated when actually needed.
|
||||
// flips from false → true. FireworksProvider is a lightweight shell; its
|
||||
// heavy children are only activated when actually needed.
|
||||
const isBootShell = !isInitialized && !isDesktopRuntime;
|
||||
|
||||
return (
|
||||
@@ -937,7 +936,6 @@ function App({ apis }: AppProps) {
|
||||
<SyncProvider key={runtimeEndpointEpoch} sdk={opencodeClient.getSdkClient()} directory={currentDirectory || ''}>
|
||||
<RuntimeAPIProvider apis={apis}>
|
||||
<FireworksProvider>
|
||||
<VoiceProvider>
|
||||
<TooltipProvider delayDuration={300} skipDelayDuration={150}>
|
||||
<div className={isDesktopRuntime ? 'h-full text-foreground bg-transparent' : 'h-full text-foreground bg-background'}>
|
||||
<SyncAppEffects embeddedBackgroundWorkEnabled={embeddedBackgroundWorkEnabled} />
|
||||
@@ -955,7 +953,6 @@ function App({ apis }: AppProps) {
|
||||
)}
|
||||
</div>
|
||||
</TooltipProvider>
|
||||
</VoiceProvider>
|
||||
</FireworksProvider>
|
||||
</RuntimeAPIProvider>
|
||||
</SyncProvider>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import React from 'react';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { BrowserVoiceButton } from '@/components/voice';
|
||||
import { ComposerDictation } from '@/components/dictation/ComposerDictation';
|
||||
// sessionStore removed — currentSessionId comes from useSessionUIStore
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
@@ -381,7 +381,7 @@ const getProjectIconColor = (projectColor?: string | null): string | undefined =
|
||||
};
|
||||
|
||||
const MemoModelControls = React.memo(ModelControls);
|
||||
const MemoBrowserVoiceButton = React.memo(BrowserVoiceButton);
|
||||
const MemoComposerDictation = React.memo(ComposerDictation);
|
||||
const MemoMobileAgentButton = React.memo(MobileAgentButton);
|
||||
const MemoMobileModelButton = React.memo(MobileModelButton);
|
||||
const MemoStatusRow = React.memo(StatusRow);
|
||||
@@ -2318,6 +2318,33 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
||||
void handleSubmitRef.current();
|
||||
}, []);
|
||||
|
||||
// Dictation: insert the transcript inline; optionally submit immediately.
|
||||
// getCurrentInputSnapshot reads textareaRef.current.value first, so setting
|
||||
// it synchronously lets handleSubmit pick up the text in the same tick.
|
||||
const handleDictationInsert = React.useCallback((text: string) => {
|
||||
setMessage((prev) => {
|
||||
const next = appendInlineText(prev, text);
|
||||
const textarea = textareaRef.current;
|
||||
if (textarea) {
|
||||
textarea.value = next;
|
||||
}
|
||||
return next;
|
||||
});
|
||||
setTimeout(() => {
|
||||
textareaRef.current?.focus();
|
||||
}, 0);
|
||||
}, []);
|
||||
|
||||
const handleDictationInsertAndSend = React.useCallback((text: string) => {
|
||||
const textarea = textareaRef.current;
|
||||
const next = appendInlineText(textarea?.value ?? messageRef.current, text);
|
||||
if (textarea) {
|
||||
textarea.value = next;
|
||||
}
|
||||
setMessage(next);
|
||||
void handleSubmitRef.current();
|
||||
}, []);
|
||||
|
||||
// Preset chips rendered outside this component (e.g. under the welcome
|
||||
// message on narrow surfaces) request a submit via the input store; consume
|
||||
// it here so it routes through the same command-aware submit path.
|
||||
@@ -4420,6 +4447,9 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
||||
: undefined}
|
||||
/>
|
||||
)}
|
||||
{/* Positioning context for the dictation overlay: covers the
|
||||
text area + footer exactly, excluding MobileSessionStatusBar. */}
|
||||
<div className={cn('relative flex flex-col', isDesktopExpanded && 'flex-1 min-h-0')}>
|
||||
<div className={cn("overflow-hidden", isDesktopExpanded && 'flex flex-1 min-h-0 flex-col')}>
|
||||
<div className="flex items-center gap-1 px-3 pt-1 flex-wrap relative z-10">
|
||||
<AttachedVSCodeFileChips onShowPopup={handleShowAttachmentPreview} />
|
||||
@@ -4559,7 +4589,16 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-x-1 flex-shrink-0">
|
||||
<MemoBrowserVoiceButton />
|
||||
<MemoComposerDictation
|
||||
radius={chatInputRadius}
|
||||
isMobile={isMobile}
|
||||
footerIconButtonClass={footerIconButtonClass}
|
||||
footerPaddingClass={footerPaddingClass}
|
||||
iconSizeClass={iconSizeClass}
|
||||
sendIconSizeClass={sendIconSizeClass}
|
||||
onInsert={handleDictationInsert}
|
||||
onInsertAndSend={handleDictationInsertAndSend}
|
||||
/>
|
||||
<ComposerActionButtons
|
||||
isMobile={isMobile}
|
||||
footerIconButtonClass={footerIconButtonClass}
|
||||
@@ -4614,7 +4653,16 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
||||
</div>
|
||||
<div className={cn('flex items-center flex-1 justify-end', footerGapClass, 'md:gap-x-3')}>
|
||||
<MemoModelControls className={cn('flex-1 min-w-0 justify-end')} />
|
||||
<MemoBrowserVoiceButton />
|
||||
<MemoComposerDictation
|
||||
radius={chatInputRadius}
|
||||
isMobile={isMobile}
|
||||
footerIconButtonClass={footerIconButtonClass}
|
||||
footerPaddingClass={footerPaddingClass}
|
||||
iconSizeClass={iconSizeClass}
|
||||
sendIconSizeClass={sendIconSizeClass}
|
||||
onInsert={handleDictationInsert}
|
||||
onInsertAndSend={handleDictationInsertAndSend}
|
||||
/>
|
||||
<ComposerActionButtons
|
||||
isMobile={isMobile}
|
||||
footerIconButtonClass={footerIconButtonClass}
|
||||
@@ -4633,6 +4681,7 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Mobile session panel: slide-up overlay toggled by MobileSessionPanelTrigger. */}
|
||||
{isMobile && <MobileSessionStatusBar />}
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
/**
|
||||
* Composer dictation controls: a mic button for the composer footer plus a
|
||||
* full-composer overlay while dictation is active (recording, transcribing,
|
||||
* or failed). The overlay mirrors the composer's own layout — the transcript
|
||||
* area uses the same paddings/typography as the textarea and the action row
|
||||
* reuses the footer icon-button styling — so toggling dictation causes no
|
||||
* vertical shift.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
|
||||
import { Icon } from '@/components/icon/Icon';
|
||||
import { useI18n } from '@/lib/i18n';
|
||||
import { useThemeSystem } from '@/contexts/useThemeSystem';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
import { useDictation } from '@/hooks/useDictation';
|
||||
import { isDictationCaptureSupported } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { isVSCodeRuntime } from '@/lib/desktop';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
import { formatShortcutForDisplay, getEffectiveShortcutCombo } from '@/lib/shortcuts';
|
||||
|
||||
interface ComposerDictationProps {
|
||||
radius?: number | string;
|
||||
isMobile: boolean;
|
||||
footerIconButtonClass: string;
|
||||
footerPaddingClass: string;
|
||||
iconSizeClass: string;
|
||||
sendIconSizeClass: string;
|
||||
disabled?: boolean;
|
||||
onInsert: (text: string) => void;
|
||||
onInsertAndSend: (text: string) => void;
|
||||
}
|
||||
|
||||
const formatDuration = (seconds: number): string => {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = seconds % 60;
|
||||
return `${mins}:${String(secs).padStart(2, '0')}`;
|
||||
};
|
||||
|
||||
const VolumeMeter: React.FC<{ volume: number }> = ({ volume }) => {
|
||||
const { currentTheme } = useThemeSystem();
|
||||
return (
|
||||
<div
|
||||
className="h-1.5 w-16 flex-shrink-0 overflow-hidden rounded-full"
|
||||
style={{ backgroundColor: currentTheme.colors.interactive.border }}
|
||||
aria-hidden="true"
|
||||
>
|
||||
<div
|
||||
className="h-full rounded-full transition-[width] duration-75"
|
||||
style={{
|
||||
width: `${Math.round(Math.min(1, volume) * 100)}%`,
|
||||
backgroundColor: currentTheme.colors.primary.base,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Polls the dictation status route while the local model is downloading and
|
||||
* returns the download percent (null while unknown / not downloading).
|
||||
*/
|
||||
const useModelDownloadProgress = (active: boolean): number | null => {
|
||||
const sttLocalModel = useConfigStore((state) => state.sttLocalModel);
|
||||
const [percent, setPercent] = React.useState<number | null>(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!active) {
|
||||
setPercent(null);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
const poll = async () => {
|
||||
try {
|
||||
const response = await runtimeFetch('/api/dictation/status', {
|
||||
query: { provider: 'local', localModel: sttLocalModel },
|
||||
});
|
||||
if (!response.ok || cancelled) {
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
const model = Array.isArray(data?.models)
|
||||
? data.models.find((m: { id: string }) => m.id === sttLocalModel)
|
||||
: null;
|
||||
if (!cancelled) {
|
||||
setPercent(typeof model?.downloadProgress === 'number' ? model.downloadProgress : null);
|
||||
}
|
||||
} catch {
|
||||
// Display-only; keep the previous value.
|
||||
}
|
||||
};
|
||||
void poll();
|
||||
const interval = setInterval(() => {
|
||||
void poll();
|
||||
}, 2000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(interval);
|
||||
};
|
||||
}, [active, sttLocalModel]);
|
||||
|
||||
return active ? percent : null;
|
||||
};
|
||||
|
||||
export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
radius,
|
||||
isMobile,
|
||||
footerIconButtonClass,
|
||||
footerPaddingClass,
|
||||
iconSizeClass,
|
||||
sendIconSizeClass,
|
||||
disabled,
|
||||
onInsert,
|
||||
onInsertAndSend,
|
||||
}) => {
|
||||
const { t } = useI18n();
|
||||
const { currentTheme } = useThemeSystem();
|
||||
const dictationEnabled = useConfigStore((state) => state.dictationEnabled);
|
||||
const shortcutOverrides = useUIStore((state) => state.shortcutOverrides);
|
||||
const dictationShortcut = formatShortcutForDisplay(getEffectiveShortcutCombo('toggle_dictation', shortcutOverrides));
|
||||
// The dictation server (WebSocket + STT worker) lives in the OpenChamber
|
||||
// web server; the VS Code bridge has no server process for it.
|
||||
const [supported] = React.useState(() => !isVSCodeRuntime() && isDictationCaptureSupported());
|
||||
|
||||
const pendingActionRef = React.useRef<'insert' | 'send' | null>(null);
|
||||
const onInsertRef = React.useRef(onInsert);
|
||||
const onInsertAndSendRef = React.useRef(onInsertAndSend);
|
||||
React.useEffect(() => {
|
||||
onInsertRef.current = onInsert;
|
||||
onInsertAndSendRef.current = onInsertAndSend;
|
||||
}, [onInsert, onInsertAndSend]);
|
||||
|
||||
const dictation = useDictation({
|
||||
onTranscript: (text) => {
|
||||
const action = pendingActionRef.current;
|
||||
pendingActionRef.current = null;
|
||||
if (action === 'send') {
|
||||
onInsertAndSendRef.current(text);
|
||||
} else {
|
||||
onInsertRef.current(text);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const {
|
||||
status,
|
||||
partialTranscript,
|
||||
volume,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
startDictation,
|
||||
confirmDictation,
|
||||
cancelDictation,
|
||||
retryFailedDictation,
|
||||
acceptPartialTranscript,
|
||||
discardFailedDictation,
|
||||
} = dictation;
|
||||
|
||||
const isModelDownloading = status === 'recording' && errorReason === 'model_download_in_progress';
|
||||
const downloadPercent = useModelDownloadProgress(isModelDownloading);
|
||||
|
||||
const statusRef = React.useRef(status);
|
||||
React.useEffect(() => {
|
||||
statusRef.current = status;
|
||||
}, [status]);
|
||||
|
||||
// Keyboard shortcut (toggle_dictation): idle -> start recording,
|
||||
// recording -> confirm and insert. Dispatched by useKeyboardShortcuts.
|
||||
React.useEffect(() => {
|
||||
const onToggle = () => {
|
||||
if (statusRef.current === 'idle') {
|
||||
void startDictation();
|
||||
} else if (statusRef.current === 'recording') {
|
||||
pendingActionRef.current = 'insert';
|
||||
void confirmDictation();
|
||||
}
|
||||
};
|
||||
window.addEventListener('openchamber:dictation-toggle', onToggle);
|
||||
return () => window.removeEventListener('openchamber:dictation-toggle', onToggle);
|
||||
}, [startDictation, confirmDictation]);
|
||||
|
||||
// While recording: Enter confirms (insert), Escape cancels. Capture-phase
|
||||
// so the composer's own Enter-to-send never fires underneath the overlay.
|
||||
React.useEffect(() => {
|
||||
if (status !== 'recording') {
|
||||
return;
|
||||
}
|
||||
const onKeyDown = (event: KeyboardEvent) => {
|
||||
if (event.isComposing) {
|
||||
return;
|
||||
}
|
||||
if (event.key === 'Enter' && !event.shiftKey && !event.metaKey && !event.ctrlKey && !event.altKey) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
pendingActionRef.current = 'insert';
|
||||
void confirmDictation();
|
||||
return;
|
||||
}
|
||||
if (event.key === 'Escape') {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
void cancelDictation();
|
||||
}
|
||||
};
|
||||
window.addEventListener('keydown', onKeyDown, { capture: true });
|
||||
return () => window.removeEventListener('keydown', onKeyDown, { capture: true });
|
||||
}, [status, confirmDictation, cancelDictation]);
|
||||
|
||||
// Pixel-parity with the composer: the real footer row is taller than our
|
||||
// buttons (its height comes from the tallest control, e.g. the model
|
||||
// picker), so measure it — it stays mounted underneath the overlay — and
|
||||
// give our action row the same height so the icons line up exactly.
|
||||
const overlayRef = React.useRef<HTMLDivElement | null>(null);
|
||||
const [footerHeight, setFooterHeight] = React.useState<number | null>(null);
|
||||
const isActiveStatus = status !== 'idle';
|
||||
React.useLayoutEffect(() => {
|
||||
if (!isActiveStatus) {
|
||||
return;
|
||||
}
|
||||
// The overlay is rendered inside the composer footer itself, so the
|
||||
// real footer is an ancestor, not a sibling.
|
||||
const realFooter = overlayRef.current?.closest<HTMLElement>('[data-chat-input-footer="true"]');
|
||||
if (realFooter && realFooter.offsetHeight > 0) {
|
||||
setFooterHeight(realFooter.offsetHeight);
|
||||
}
|
||||
}, [isActiveStatus]);
|
||||
|
||||
if (!supported || !dictationEnabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const isActive = status !== 'idle';
|
||||
|
||||
const confirmWith = (action: 'insert' | 'send') => {
|
||||
pendingActionRef.current = action;
|
||||
void confirmDictation();
|
||||
};
|
||||
|
||||
const retry = () => {
|
||||
pendingActionRef.current = 'insert';
|
||||
void retryFailedDictation();
|
||||
};
|
||||
|
||||
const placeholderText = (() => {
|
||||
if (status === 'failed') {
|
||||
return '';
|
||||
}
|
||||
if (status === 'uploading') {
|
||||
return t('chat.dictation.processing');
|
||||
}
|
||||
if (isModelDownloading) {
|
||||
return downloadPercent !== null
|
||||
? t('chat.dictation.downloadingModelProgress', { percent: String(downloadPercent) })
|
||||
: t('chat.dictation.downloadingModel');
|
||||
}
|
||||
return t('chat.dictation.listening');
|
||||
})();
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
className={footerIconButtonClass}
|
||||
onClick={() => {
|
||||
void startDictation();
|
||||
}}
|
||||
disabled={disabled || isActive}
|
||||
title={dictationShortcut ? `${t('chat.dictation.start')} (${dictationShortcut})` : t('chat.dictation.start')}
|
||||
aria-label={t('chat.dictation.start')}
|
||||
>
|
||||
<Icon name="mic" className={cn(iconSizeClass, 'text-current')} />
|
||||
</button>
|
||||
{isActive ? (
|
||||
<div
|
||||
ref={overlayRef}
|
||||
// overflow-x/y split on purpose: mobile.css rewrites the
|
||||
// shorthand `.overflow-hidden` to overflow-y:auto on touch
|
||||
// devices, which painted a phantom scrollbar on Android.
|
||||
className="absolute inset-0 z-50 flex flex-col overflow-x-hidden overflow-y-hidden"
|
||||
style={{
|
||||
borderRadius: radius,
|
||||
// Must match the composer box background exactly so the
|
||||
// overlay reads as the same surface, not a layer on top.
|
||||
backgroundColor: currentTheme.colors.surface.subtle,
|
||||
}}
|
||||
role="dialog"
|
||||
aria-label={t('chat.dictation.overlayAria')}
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
// Text paddings match the composer textarea, plus the
|
||||
// 4px (pt-1) attachment-chips row that always renders
|
||||
// above it: desktop 16+4px, mobile 10+4px from the top.
|
||||
// min-h-0 (not a fixed min height): the mobile composer
|
||||
// is shorter than 52px of text area + footer, and a
|
||||
// fixed min pushed the action row 4px below the real
|
||||
// footer. The area must shrink to whatever space the
|
||||
// underlying composer actually has.
|
||||
'flex-1 min-h-0 overflow-y-auto px-3',
|
||||
isMobile ? 'pt-3.5 pb-2.5' : 'pt-5 pb-2',
|
||||
)}
|
||||
>
|
||||
{partialTranscript ? (
|
||||
<p className="typography-markdown md:typography-ui-label whitespace-pre-wrap" style={{ color: currentTheme.colors.surface.foreground }}>
|
||||
{partialTranscript}
|
||||
</p>
|
||||
) : (
|
||||
<p className="typography-markdown md:typography-ui-label" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||
{placeholderText}
|
||||
</p>
|
||||
)}
|
||||
{status === 'failed' ? (
|
||||
<p className="typography-meta mt-1" style={{ color: currentTheme.colors.status.error }}>
|
||||
{error || t('chat.dictation.failed')}
|
||||
</p>
|
||||
) : null}
|
||||
{status === 'recording' && error && !isModelDownloading ? (
|
||||
<p className="typography-meta mt-1" style={{ color: currentTheme.colors.status.warning }}>
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
<div
|
||||
className={cn('flex flex-shrink-0 items-center gap-x-3', footerPaddingClass)}
|
||||
style={footerHeight ? { height: footerHeight } : undefined}
|
||||
>
|
||||
{status === 'recording' ? (
|
||||
<>
|
||||
<span className="relative ml-1 flex h-2 w-2 flex-shrink-0" aria-hidden="true">
|
||||
<span
|
||||
className="absolute inline-flex h-full w-full animate-ping rounded-full opacity-60"
|
||||
style={{ backgroundColor: currentTheme.colors.status.error }}
|
||||
/>
|
||||
<span
|
||||
className="relative inline-flex h-2 w-2 rounded-full"
|
||||
style={{ backgroundColor: currentTheme.colors.status.error }}
|
||||
/>
|
||||
</span>
|
||||
<VolumeMeter volume={volume} />
|
||||
<span className="typography-meta tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||
{formatDuration(duration)}
|
||||
</span>
|
||||
</>
|
||||
) : status === 'uploading' ? (
|
||||
<Icon name="loader-4" className="ml-1 h-4 w-4 animate-spin" style={{ color: currentTheme.colors.surface.mutedForeground }} />
|
||||
) : null}
|
||||
{/* Same inter-control gap as the composer's right cluster:
|
||||
gap-x-1 on mobile, md:gap-x-3 on desktop. */}
|
||||
<div className={cn('ml-auto flex items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
|
||||
{status === 'recording' ? (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
|
||||
onClick={() => {
|
||||
void cancelDictation();
|
||||
}}
|
||||
title={t('chat.dictation.cancel')}
|
||||
aria-label={t('chat.dictation.cancel')}
|
||||
>
|
||||
<Icon name="close" className={iconSizeClass} />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={footerIconButtonClass}
|
||||
onClick={() => confirmWith('insert')}
|
||||
title={t('chat.dictation.insert')}
|
||||
aria-label={t('chat.dictation.insert')}
|
||||
>
|
||||
<Icon name="check" className={iconSizeClass} />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(footerIconButtonClass, 'text-primary hover:text-primary')}
|
||||
onClick={() => confirmWith('send')}
|
||||
title={t('chat.dictation.insertAndSend')}
|
||||
aria-label={t('chat.dictation.insertAndSend')}
|
||||
>
|
||||
<Icon name="send-plane-2" className={sendIconSizeClass} />
|
||||
</button>
|
||||
</>
|
||||
) : status === 'uploading' ? (
|
||||
<button
|
||||
type="button"
|
||||
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
|
||||
onClick={() => {
|
||||
void cancelDictation();
|
||||
}}
|
||||
title={t('chat.dictation.cancel')}
|
||||
aria-label={t('chat.dictation.cancel')}
|
||||
>
|
||||
<Icon name="close" className={iconSizeClass} />
|
||||
</button>
|
||||
) : (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
|
||||
onClick={discardFailedDictation}
|
||||
title={t('chat.dictation.discard')}
|
||||
aria-label={t('chat.dictation.discard')}
|
||||
>
|
||||
<Icon name="close" className={iconSizeClass} />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={footerIconButtonClass}
|
||||
onClick={retry}
|
||||
title={t('chat.dictation.retry')}
|
||||
aria-label={t('chat.dictation.retry')}
|
||||
>
|
||||
<Icon name="refresh" className={iconSizeClass} />
|
||||
</button>
|
||||
{partialTranscript.trim() ? (
|
||||
<button
|
||||
type="button"
|
||||
className={footerIconButtonClass}
|
||||
onClick={() => {
|
||||
pendingActionRef.current = 'insert';
|
||||
acceptPartialTranscript();
|
||||
}}
|
||||
title={t('chat.dictation.insert')}
|
||||
aria-label={t('chat.dictation.insert')}
|
||||
>
|
||||
<Icon name="check" className={iconSizeClass} />
|
||||
</button>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -7,7 +7,6 @@ import type { ThemeMode } from '@/types/theme';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
import { useMessageQueueStore, type FollowUpBehavior } from '@/stores/messageQueueStore';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { isCapacitorApp } from '@/lib/platform';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import { NumberInput } from '@/components/ui/number-input';
|
||||
@@ -322,10 +321,7 @@ export const OpenChamberVisualSettings: React.FC<OpenChamberVisualSettingsProps>
|
||||
const setShowSplitAssistantMessageActions = useUIStore(state => state.setShowSplitAssistantMessageActions);
|
||||
const messageStreamTransport = useConfigStore((state) => state.settingsMessageStreamTransport);
|
||||
const setMessageStreamTransport = useConfigStore((state) => state.setSettingsMessageStreamTransport);
|
||||
// Capacitor apps are locked to SSE (native WebSocket streaming is unreliable on mobile);
|
||||
// sync-context forces it too. Show SSE selected and disable the other options here.
|
||||
const isCapacitorAppRuntime = React.useMemo(() => isCapacitorApp(), []);
|
||||
const effectiveMessageStreamTransport = isCapacitorAppRuntime ? 'sse' : messageStreamTransport;
|
||||
const effectiveMessageStreamTransport = messageStreamTransport;
|
||||
const settingsDefaultFileViewerPreview = useConfigStore((state) => state.settingsDefaultFileViewerPreview);
|
||||
const setSettingsDefaultFileViewerPreview = useConfigStore((state) => state.setSettingsDefaultFileViewerPreview);
|
||||
const isSettingsDialogOpen = useUIStore(state => state.isSettingsDialogOpen);
|
||||
@@ -1531,7 +1527,6 @@ export const OpenChamberVisualSettings: React.FC<OpenChamberVisualSettingsProps>
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={effectiveMessageStreamTransport === option.id}
|
||||
disabled={isCapacitorAppRuntime && option.id !== 'sse'}
|
||||
className="!font-normal"
|
||||
onClick={() => handleMessageStreamTransportChange(option.id)}
|
||||
>
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import React, { useEffect, useState, useCallback, useMemo } from 'react';
|
||||
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useDeviceInfo } from '@/lib/device';
|
||||
|
||||
@@ -11,103 +10,362 @@ import {
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import { Radio } from '@/components/ui/radio';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { NumberInput } from '@/components/ui/number-input';
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import { Icon } from "@/components/icon/Icon";
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { audioStreamService } from '@/lib/voice/audioStreamService';
|
||||
import { wasmSttService, WASM_MODELS } from '@/lib/voice/wasmSttService';
|
||||
import type { WasmModelStatus } from '@/lib/voice/wasmSttService';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
import { useI18n } from '@/lib/i18n';
|
||||
import { useLocalTTS } from '@/hooks/useLocalTTS';
|
||||
import { disposePreviewAudio } from './voicePreviewAudio';
|
||||
const LANGUAGE_OPTIONS = [
|
||||
{ value: 'en-US', label: 'English' },
|
||||
{ value: 'es-ES', label: 'Español' },
|
||||
{ value: 'fr-FR', label: 'Français' },
|
||||
{ value: 'de-DE', label: 'Deutsch' },
|
||||
{ value: 'ja-JP', label: '日本語' },
|
||||
{ value: 'zh-CN', label: '中文' },
|
||||
{ value: 'pt-BR', label: 'Português' },
|
||||
{ value: 'it-IT', label: 'Italiano' },
|
||||
{ value: 'ko-KR', label: '한국어' },
|
||||
{ value: 'uk-UA', label: 'Українська' },
|
||||
];
|
||||
|
||||
const WasmModelStatusIndicator = ({ modelId }: { modelId: string }) => {
|
||||
const LOCAL_STT_MODELS = [
|
||||
{
|
||||
id: 'parakeet-tdt-0.6b-v2-int8',
|
||||
labelKey: 'settings.voice.page.stt.model.parakeetV2',
|
||||
badgeKey: 'settings.voice.page.stt.badge.bestForEnglish',
|
||||
accuracy: 5,
|
||||
speed: 5,
|
||||
size: '460 MB',
|
||||
},
|
||||
{
|
||||
id: 'parakeet-tdt-0.6b-v3-int8',
|
||||
labelKey: 'settings.voice.page.stt.model.parakeetV3',
|
||||
badgeKey: 'settings.voice.page.stt.badge.bestForMultilingual',
|
||||
accuracy: 5,
|
||||
speed: 4,
|
||||
size: '465 MB',
|
||||
},
|
||||
{
|
||||
id: 'whisper-base-int8',
|
||||
labelKey: 'settings.voice.page.stt.model.whisperBase',
|
||||
badgeKey: null,
|
||||
accuracy: 3,
|
||||
speed: 3,
|
||||
size: '200 MB',
|
||||
},
|
||||
{
|
||||
id: 'whisper-tiny-int8',
|
||||
labelKey: 'settings.voice.page.stt.model.whisperTiny',
|
||||
badgeKey: null,
|
||||
accuracy: 2,
|
||||
speed: 4,
|
||||
size: '115 MB',
|
||||
},
|
||||
] as const;
|
||||
|
||||
interface DictationModelState {
|
||||
id: string;
|
||||
installed: boolean;
|
||||
downloading: boolean;
|
||||
downloadProgress: number | null;
|
||||
downloadError: string | null;
|
||||
}
|
||||
|
||||
const RatingBar = ({ value, label }: { value: number; label: string }) => (
|
||||
<span className="flex items-center gap-1.5" title={`${label}: ${value}/5`}>
|
||||
<span
|
||||
className="h-1.5 w-12 flex-shrink-0 overflow-hidden rounded-full bg-[var(--interactive-border)]"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<span
|
||||
className="block h-full rounded-full bg-[var(--primary-base)]"
|
||||
style={{ width: `${(value / 5) * 100}%` }}
|
||||
/>
|
||||
</span>
|
||||
<span className="typography-ui-compact text-muted-foreground">{label}</span>
|
||||
</span>
|
||||
);
|
||||
|
||||
const LocalModelPicker = ({
|
||||
selectedModelId,
|
||||
onSelect,
|
||||
}: {
|
||||
selectedModelId: string;
|
||||
onSelect: (modelId: string) => void;
|
||||
}) => {
|
||||
const { t } = useI18n();
|
||||
const [status, setStatus] = useState<WasmModelStatus>(wasmSttService.getModelStatus());
|
||||
const [loading, setLoading] = useState(false);
|
||||
const tUnsafe = useCallback((key: string) => t(key as Parameters<typeof t>[0]), [t]);
|
||||
const [models, setModels] = useState<Map<string, DictationModelState>>(new Map());
|
||||
const [requestingId, setRequestingId] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (s: WasmModelStatus) => setStatus(s);
|
||||
wasmSttService.onModelStatusChange = handler;
|
||||
return () => { wasmSttService.onModelStatusChange = null; };
|
||||
const refresh = useCallback(async () => {
|
||||
try {
|
||||
const response = await runtimeFetch('/api/dictation/status', {
|
||||
query: { provider: 'local' },
|
||||
});
|
||||
if (!response.ok) {
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (Array.isArray(data?.models)) {
|
||||
setModels(new Map(data.models.map((m: DictationModelState) => [m.id, m])));
|
||||
}
|
||||
} catch {
|
||||
// Display-only status; keep the previous state on fetch failure.
|
||||
}
|
||||
}, []);
|
||||
|
||||
const currentModelId = wasmSttService.getCurrentModelId();
|
||||
const isLoadingOrDownloading = status.state === 'downloading' || status.state === 'loading';
|
||||
|
||||
// Reset local loading state when model finishes loading or errors
|
||||
useEffect(() => {
|
||||
if (status.state !== 'downloading' && status.state !== 'loading') {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [status.state]);
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
const handleDownload = async () => {
|
||||
setLoading(true);
|
||||
const anyDownloading = Array.from(models.values()).some((m) => m.downloading);
|
||||
useEffect(() => {
|
||||
if (!anyDownloading) {
|
||||
return;
|
||||
}
|
||||
const interval = setInterval(() => {
|
||||
void refresh();
|
||||
}, 2000);
|
||||
return () => clearInterval(interval);
|
||||
}, [anyDownloading, refresh]);
|
||||
|
||||
const handleDownload = async (modelId: string) => {
|
||||
setRequestingId(modelId);
|
||||
try {
|
||||
await wasmSttService.loadModel(modelId);
|
||||
await runtimeFetch(`/api/dictation/models/${encodeURIComponent(modelId)}/download`, {
|
||||
method: 'POST',
|
||||
});
|
||||
await refresh();
|
||||
} catch {
|
||||
// Error is shown via status indicator
|
||||
// Status refresh reports errors.
|
||||
} finally {
|
||||
setRequestingId(null);
|
||||
}
|
||||
};
|
||||
|
||||
if (status.state === 'ready' && currentModelId === modelId) {
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="typography-ui-compact text-green-600 dark:text-green-400">
|
||||
{t('settings.voice.page.stt.wasmLoaded')}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const handleDelete = async (modelId: string) => {
|
||||
setRequestingId(modelId);
|
||||
try {
|
||||
await runtimeFetch(`/api/dictation/models/${encodeURIComponent(modelId)}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
await refresh();
|
||||
} catch {
|
||||
// Status refresh reports errors.
|
||||
} finally {
|
||||
setRequestingId(null);
|
||||
}
|
||||
};
|
||||
|
||||
if (isLoadingOrDownloading) {
|
||||
const progress = status.state === 'downloading' ? Math.round(status.progress) : undefined;
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="typography-ui-compact text-muted-foreground">
|
||||
{status.state === 'downloading' ? t('settings.voice.page.stt.wasmDownloading') : t('settings.voice.page.stt.wasmLoading')}
|
||||
{progress !== undefined ? ` (${progress}%)` : ''}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div role="radiogroup" aria-label={t('settings.voice.page.field.model')} className="mt-1 space-y-0">
|
||||
{LOCAL_STT_MODELS.map((entry) => {
|
||||
const selected = selectedModelId === entry.id;
|
||||
const state = models.get(entry.id) ?? null;
|
||||
return (
|
||||
<div key={entry.id} className="flex w-full items-start gap-2 py-1.5">
|
||||
<div className="pt-0.5">
|
||||
<Radio
|
||||
checked={selected}
|
||||
onChange={() => onSelect(entry.id)}
|
||||
ariaLabel={tUnsafe(entry.labelKey)}
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className="min-w-0 flex-1 cursor-pointer"
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onClick={() => onSelect(entry.id)}
|
||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); onSelect(entry.id); } }}
|
||||
>
|
||||
<div className="flex flex-wrap items-center gap-x-2 gap-y-0.5">
|
||||
<span className={cn('typography-ui-label font-normal', selected ? 'text-foreground' : 'text-foreground/50')}>
|
||||
{tUnsafe(entry.labelKey)}
|
||||
</span>
|
||||
{entry.badgeKey ? (
|
||||
<span
|
||||
className="rounded px-1 text-[9px] font-medium uppercase leading-[14px] tracking-wide"
|
||||
style={{
|
||||
backgroundColor: 'var(--status-success-background)',
|
||||
color: 'var(--status-success)',
|
||||
border: '1px solid var(--status-success-border)',
|
||||
}}
|
||||
>
|
||||
{tUnsafe(entry.badgeKey)}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="mt-0.5 flex flex-wrap items-center gap-x-4 gap-y-0.5">
|
||||
<RatingBar value={entry.accuracy} label={t('settings.voice.page.stt.meta.accuracy')} />
|
||||
<RatingBar value={entry.speed} label={t('settings.voice.page.stt.meta.speed')} />
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">{entry.size}</span>
|
||||
</div>
|
||||
{state?.downloadError ? (
|
||||
<p className="mt-0.5 typography-meta text-[var(--status-error)]">{state.downloadError}</p>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="flex w-14 flex-shrink-0 items-center justify-end gap-1 pt-0.5">
|
||||
{state?.installed ? (
|
||||
<>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
|
||||
disabled={requestingId === entry.id}
|
||||
onClick={() => { void handleDelete(entry.id); }}
|
||||
title={t('settings.voice.page.stt.modelDelete')}
|
||||
aria-label={t('settings.voice.page.stt.modelDelete')}
|
||||
>
|
||||
<Icon name="delete-bin" className="h-4 w-4" />
|
||||
</Button>
|
||||
<Icon
|
||||
name="checkbox-circle"
|
||||
className="h-4 w-4 flex-shrink-0 text-[var(--status-success)]"
|
||||
aria-label={t('settings.voice.page.stt.modelInstalled')}
|
||||
/>
|
||||
</>
|
||||
) : state?.downloading ? (
|
||||
<span className="flex items-center gap-1.5">
|
||||
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">
|
||||
{typeof state.downloadProgress === 'number' ? `${state.downloadProgress}%` : ''}
|
||||
</span>
|
||||
</span>
|
||||
) : (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0"
|
||||
disabled={requestingId === entry.id}
|
||||
onClick={() => { void handleDownload(entry.id); }}
|
||||
title={t('settings.voice.page.stt.modelDownload')}
|
||||
aria-label={t('settings.voice.page.stt.modelDownload')}
|
||||
>
|
||||
<Icon name="download" className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
if (status.state === 'error') {
|
||||
// Partial download (cached progress): show retry button.
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="typography-ui-compact text-destructive">{status.error}</span>
|
||||
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
|
||||
{t('settings.voice.page.stt.wasmRetry')}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
/** Kokoro en_v0_19 speaker ids in sherpa-onnx order. */
|
||||
const KOKORO_VOICE_OPTIONS = [
|
||||
{ id: 0, label: 'Alloy (af)' },
|
||||
{ id: 1, label: 'Bella (af)' },
|
||||
{ id: 2, label: 'Nicole (af)' },
|
||||
{ id: 3, label: 'Sarah (af)' },
|
||||
{ id: 4, label: 'Sky (af)' },
|
||||
{ id: 5, label: 'Adam (am)' },
|
||||
{ id: 6, label: 'Michael (am)' },
|
||||
{ id: 7, label: 'Emma (bf)' },
|
||||
{ id: 8, label: 'Isabella (bf)' },
|
||||
{ id: 9, label: 'George (bm)' },
|
||||
{ id: 10, label: 'Lewis (bm)' },
|
||||
];
|
||||
|
||||
const LOCAL_TTS_MODEL_ID = 'kokoro-en-v0_19';
|
||||
|
||||
const LocalTtsModelStatus = () => {
|
||||
const { t } = useI18n();
|
||||
const [model, setModel] = useState<DictationModelState | null>(null);
|
||||
const [requesting, setRequesting] = useState(false);
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
try {
|
||||
const response = await runtimeFetch('/api/dictation/status', { query: { provider: 'local' } });
|
||||
if (!response.ok) {
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
const entry = Array.isArray(data?.ttsModels)
|
||||
? data.ttsModels.find((m: DictationModelState) => m.id === LOCAL_TTS_MODEL_ID)
|
||||
: null;
|
||||
if (entry) {
|
||||
setModel(entry);
|
||||
}
|
||||
} catch {
|
||||
// Display-only status; keep the previous state on fetch failure.
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!model?.downloading) {
|
||||
return;
|
||||
}
|
||||
const interval = setInterval(() => {
|
||||
void refresh();
|
||||
}, 2000);
|
||||
return () => clearInterval(interval);
|
||||
}, [model?.downloading, refresh]);
|
||||
|
||||
const request = async (method: 'POST' | 'DELETE') => {
|
||||
setRequesting(true);
|
||||
try {
|
||||
const path = method === 'POST'
|
||||
? `/api/dictation/models/${LOCAL_TTS_MODEL_ID}/download`
|
||||
: `/api/dictation/models/${LOCAL_TTS_MODEL_ID}`;
|
||||
await runtimeFetch(path, { method });
|
||||
await refresh();
|
||||
} catch {
|
||||
// Status refresh reports errors.
|
||||
} finally {
|
||||
setRequesting(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (!model) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="typography-ui-compact text-muted-foreground">
|
||||
{t('settings.voice.page.stt.wasmNotLoaded')}
|
||||
</span>
|
||||
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
|
||||
{t('settings.voice.page.stt.wasmDownload')}
|
||||
</Button>
|
||||
<div className="flex items-center gap-2 py-1.5">
|
||||
<span className="typography-ui-label text-foreground">Kokoro</span>
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">305 MB</span>
|
||||
{model.installed ? (
|
||||
<>
|
||||
<Icon
|
||||
name="checkbox-circle"
|
||||
className="h-4 w-4 text-[var(--status-success)]"
|
||||
aria-label={t('settings.voice.page.stt.modelInstalled')}
|
||||
/>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
|
||||
disabled={requesting}
|
||||
onClick={() => { void request('DELETE'); }}
|
||||
title={t('settings.voice.page.stt.modelDelete')}
|
||||
aria-label={t('settings.voice.page.stt.modelDelete')}
|
||||
>
|
||||
<Icon name="delete-bin" className="h-4 w-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : model.downloading ? (
|
||||
<span className="flex items-center gap-1.5">
|
||||
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">
|
||||
{typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''}
|
||||
</span>
|
||||
</span>
|
||||
) : (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0"
|
||||
disabled={requesting}
|
||||
onClick={() => { void request('POST'); }}
|
||||
title={t('settings.voice.page.stt.modelDownload')}
|
||||
aria-label={t('settings.voice.page.stt.modelDownload')}
|
||||
>
|
||||
<Icon name="download" className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
{model.downloadError ? (
|
||||
<span className="typography-meta text-[var(--status-error)]">{model.downloadError}</span>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -131,11 +389,6 @@ const OPENAI_VOICE_OPTIONS = [
|
||||
export const VoiceSettings: React.FC = () => {
|
||||
const { t } = useI18n();
|
||||
const { isMobile } = useDeviceInfo();
|
||||
const {
|
||||
isSupported,
|
||||
language,
|
||||
setLanguage,
|
||||
} = useBrowserVoice();
|
||||
const voiceProvider = useConfigStore((state) => state.voiceProvider);
|
||||
const setVoiceProvider = useConfigStore((state) => state.setVoiceProvider);
|
||||
const speechRate = useConfigStore((state) => state.speechRate);
|
||||
@@ -146,6 +399,22 @@ export const VoiceSettings: React.FC = () => {
|
||||
const setSpeechVolume = useConfigStore((state) => state.setSpeechVolume);
|
||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||
const setSayVoice = useConfigStore((state) => state.setSayVoice);
|
||||
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
|
||||
const setLocalTtsVoiceId = useConfigStore((state) => state.setLocalTtsVoiceId);
|
||||
const { speak: speakLocalTts, stop: stopLocalTts, isPlaying: isLocalTtsPlaying, error: localTtsError } = useLocalTTS();
|
||||
|
||||
const previewLocalVoice = useCallback(() => {
|
||||
if (isLocalTtsPlaying) {
|
||||
stopLocalTts();
|
||||
return;
|
||||
}
|
||||
const voiceLabel = KOKORO_VOICE_OPTIONS.find((v) => v.id === localTtsVoiceId)?.label
|
||||
?? String(localTtsVoiceId);
|
||||
void speakLocalTts(t('settings.voice.page.preview.voiceLine', { voiceName: voiceLabel }), {
|
||||
speakerId: localTtsVoiceId,
|
||||
speed: useConfigStore.getState().speechRate,
|
||||
});
|
||||
}, [isLocalTtsPlaying, localTtsVoiceId, speakLocalTts, stopLocalTts, t]);
|
||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||
const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice);
|
||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||
@@ -172,19 +441,13 @@ export const VoiceSettings: React.FC = () => {
|
||||
const setSttApiKey = useConfigStore((state) => state.setSttApiKey);
|
||||
const sttModel = useConfigStore((state) => state.sttModel);
|
||||
const setSttModel = useConfigStore((state) => state.setSttModel);
|
||||
const wasmSttModel = useConfigStore((state) => state.wasmSttModel);
|
||||
const setWasmSttModel = useConfigStore((state) => state.setWasmSttModel);
|
||||
const sttLocalModel = useConfigStore((state) => state.sttLocalModel);
|
||||
const setSttLocalModel = useConfigStore((state) => state.setSttLocalModel);
|
||||
const sttLanguage = useConfigStore((state) => state.sttLanguage);
|
||||
const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
|
||||
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
|
||||
const setSttSilenceThresholdDb = useConfigStore((state) => state.setSttSilenceThresholdDb);
|
||||
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
|
||||
const setSttSilenceHoldMs = useConfigStore((state) => state.setSttSilenceHoldMs);
|
||||
const sttTranscribeOnStop = useConfigStore((state) => state.sttTranscribeOnStop);
|
||||
const setSttTranscribeOnStop = useConfigStore((state) => state.setSttTranscribeOnStop);
|
||||
const setShowMessageTTSButtons = useConfigStore((state) => state.setShowMessageTTSButtons);
|
||||
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
|
||||
const setVoiceModeEnabled = useConfigStore((state) => state.setVoiceModeEnabled);
|
||||
const dictationEnabled = useConfigStore((state) => state.dictationEnabled);
|
||||
const setDictationEnabled = useConfigStore((state) => state.setDictationEnabled);
|
||||
|
||||
const [isSayAvailable, setIsSayAvailable] = useState(false);
|
||||
const [sayVoices, setSayVoices] = useState<Array<{ name: string; locale: string }>>([]);
|
||||
@@ -274,7 +537,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
}, [isBrowserPreviewPlaying]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceModeEnabled || (voiceProvider !== 'openai' && voiceProvider !== 'openai-compatible')) {
|
||||
if (!showMessageTTSButtons || (voiceProvider !== 'openai' && voiceProvider !== 'openai-compatible')) {
|
||||
setIsOpenAIAvailable(openaiApiKey.trim().length > 0);
|
||||
return;
|
||||
}
|
||||
@@ -292,10 +555,10 @@ export const VoiceSettings: React.FC = () => {
|
||||
};
|
||||
|
||||
checkOpenAIAvailability();
|
||||
}, [openaiApiKey, voiceModeEnabled, voiceProvider]);
|
||||
}, [openaiApiKey, showMessageTTSButtons, voiceProvider]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceModeEnabled) {
|
||||
if (!showMessageTTSButtons) {
|
||||
setIsSayAvailable(false);
|
||||
setSayVoices([]);
|
||||
return;
|
||||
@@ -317,7 +580,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
.catch(() => {
|
||||
setIsSayAvailable(false);
|
||||
});
|
||||
}, [voiceModeEnabled]);
|
||||
}, [showMessageTTSButtons]);
|
||||
|
||||
const previewVoice = useCallback(async () => {
|
||||
if (previewAudio) {
|
||||
@@ -498,11 +761,11 @@ export const VoiceSettings: React.FC = () => {
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
|
||||
{/* Voice Setup */}
|
||||
<div data-settings-item="voice.voice-setup" className="mb-8">
|
||||
{/* Playback (read messages aloud) */}
|
||||
<div data-settings-item="voice.playback" className="mb-8">
|
||||
<div className="mb-1 px-1">
|
||||
<h3 className="typography-ui-header font-medium text-foreground">
|
||||
{t('settings.voice.page.section.voiceSetup')}
|
||||
{t('settings.voice.page.section.playbackAndSummary')}
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
@@ -512,15 +775,15 @@ export const VoiceSettings: React.FC = () => {
|
||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
aria-pressed={voiceModeEnabled}
|
||||
onClick={() => setVoiceModeEnabled(!voiceModeEnabled)}
|
||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setVoiceModeEnabled(!voiceModeEnabled); } }}
|
||||
aria-pressed={showMessageTTSButtons}
|
||||
onClick={() => setShowMessageTTSButtons(!showMessageTTSButtons)}
|
||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setShowMessageTTSButtons(!showMessageTTSButtons); } }}
|
||||
>
|
||||
<Checkbox checked={voiceModeEnabled} onChange={setVoiceModeEnabled} ariaLabel={t('settings.voice.page.field.enableVoiceModeAria')} />
|
||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.enableVoiceMode')}</span>
|
||||
<Checkbox checked={showMessageTTSButtons} onChange={setShowMessageTTSButtons} ariaLabel={t('settings.voice.page.field.messageReadAloudButtonAria')} />
|
||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.messageReadAloudButton')}</span>
|
||||
</div>
|
||||
|
||||
{voiceModeEnabled && (
|
||||
{showMessageTTSButtons && (
|
||||
<>
|
||||
<div className="pb-1.5 pt-0.5">
|
||||
<div className="flex min-w-0 flex-col gap-1.5">
|
||||
@@ -533,6 +796,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
<TooltipContent sideOffset={8} className="max-w-xs">
|
||||
<ul className="space-y-1">
|
||||
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.browser')}</li>
|
||||
<li><strong>{t('settings.voice.page.provider.local')}</strong> {t('settings.voice.page.tooltip.localTts')}</li>
|
||||
<li><strong>OpenAI:</strong> {t('settings.voice.page.tooltip.openai')}</li>
|
||||
<li><strong>{t('settings.voice.page.provider.custom')}</strong> {t('settings.voice.page.tooltip.custom')}</li>
|
||||
<li><strong>{t('settings.voice.page.provider.say')}</strong> {t('settings.voice.page.tooltip.say')}</li>
|
||||
@@ -550,6 +814,15 @@ export const VoiceSettings: React.FC = () => {
|
||||
>
|
||||
{t('settings.voice.page.provider.browser')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={voiceProvider === 'local'}
|
||||
onClick={() => setVoiceProvider('local')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.provider.local')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
@@ -706,10 +979,39 @@ export const VoiceSettings: React.FC = () => {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Local (Kokoro) TTS model status */}
|
||||
{voiceProvider === 'local' && <LocalTtsModelStatus />}
|
||||
|
||||
{/* Voice Selection */}
|
||||
<div className="flex items-center gap-8 py-1.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.voice')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{voiceProvider === 'local' && (
|
||||
<>
|
||||
<Select
|
||||
value={String(localTtsVoiceId)}
|
||||
onValueChange={(value) => setLocalTtsVoiceId(Number.parseInt(value, 10) || 0)}
|
||||
>
|
||||
<SelectTrigger className="w-fit">
|
||||
<SelectValue placeholder={t('settings.voice.page.field.selectVoicePlaceholder')}>
|
||||
{(value) => KOKORO_VOICE_OPTIONS.find((v) => String(v.id) === value)?.label ?? value}
|
||||
</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{KOKORO_VOICE_OPTIONS.map((v) => (
|
||||
<SelectItem key={v.id} value={String(v.id)}>{v.label}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button size="xs" variant="ghost" onClick={previewLocalVoice} title={t('settings.voice.page.actions.preview')}>
|
||||
{isLocalTtsPlaying ? <Icon name="stop" className="w-3.5 h-3.5" /> : <Icon name="play" className="w-3.5 h-3.5" />}
|
||||
</Button>
|
||||
{localTtsError ? (
|
||||
<span className="typography-meta text-[var(--status-error)]">{localTtsError}</span>
|
||||
) : null}
|
||||
</>
|
||||
)}
|
||||
|
||||
{voiceProvider === 'openai' && isOpenAIAvailable && (
|
||||
<>
|
||||
<Select value={openaiVoice} onValueChange={setOpenaiVoice}>
|
||||
@@ -775,7 +1077,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
<div className="flex items-center gap-8 py-1.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechRate')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechRate} onChange={(e) => setSpeechRate(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
|
||||
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechRate} onChange={(e) => setSpeechRate(Number(e.target.value))} className={sliderClass} />}
|
||||
<NumberInput value={speechRate} onValueChange={setSpeechRate} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
|
||||
</div>
|
||||
</div>
|
||||
@@ -784,7 +1086,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
<div className="flex items-center gap-8 py-1.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechPitch')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechPitch} onChange={(e) => setSpeechPitch(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
|
||||
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechPitch} onChange={(e) => setSpeechPitch(Number(e.target.value))} className={sliderClass} />}
|
||||
<NumberInput value={speechPitch} onValueChange={setSpeechPitch} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
|
||||
</div>
|
||||
</div>
|
||||
@@ -793,7 +1095,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
<div className="flex items-center gap-8 py-1.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechVolume')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={0} max={1} step={0.1} value={speechVolume} onChange={(e) => setSpeechVolume(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
|
||||
{!isMobile && <input type="range" min={0} max={1} step={0.1} value={speechVolume} onChange={(e) => setSpeechVolume(Number(e.target.value))} className={sliderClass} />}
|
||||
{isMobile ? (
|
||||
<NumberInput value={Math.round(speechVolume * 100)} onValueChange={(v) => setSpeechVolume(v / 100)} min={0} max={100} step={10} className="w-16 tabular-nums" />
|
||||
) : (
|
||||
@@ -804,20 +1106,34 @@ export const VoiceSettings: React.FC = () => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Language */}
|
||||
<div className="flex items-center gap-8 py-1.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.language')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
<Select value={language} onValueChange={setLanguage} disabled={!isSupported}>
|
||||
<SelectTrigger className="w-fit">
|
||||
<SelectValue placeholder={t('settings.voice.page.field.selectLanguagePlaceholder')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{LANGUAGE_OPTIONS.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value}>{lang.label}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{/* TTS input mode */}
|
||||
<div className="pb-1.5 pt-0.5">
|
||||
<div className="flex min-w-0 flex-col gap-1.5">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="typography-ui-label text-foreground">
|
||||
{t('settings.voice.page.field.ttsInputMode')}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={ttsInputMode === 'sanitized'}
|
||||
onClick={() => setTtsInputMode('sanitized')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.field.ttsInputModeSanitized')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={ttsInputMode === 'raw'}
|
||||
onClick={() => setTtsInputMode('raw')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.field.ttsInputModeRaw')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
@@ -826,8 +1142,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
</div>
|
||||
|
||||
{/* Speech Recognition */}
|
||||
{voiceModeEnabled && (
|
||||
<div data-settings-item="voice.speech-recognition" className="mb-8">
|
||||
<div data-settings-item="voice.speech-recognition" className="mb-8">
|
||||
<div className="mb-1 px-1">
|
||||
<h3 className="typography-ui-header font-medium text-foreground">
|
||||
{t('settings.voice.page.section.speechRecognition')}
|
||||
@@ -835,6 +1150,19 @@ export const VoiceSettings: React.FC = () => {
|
||||
</div>
|
||||
|
||||
<section className="px-2 pb-2 pt-0 space-y-0">
|
||||
<div
|
||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
aria-pressed={dictationEnabled}
|
||||
onClick={() => setDictationEnabled(!dictationEnabled)}
|
||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setDictationEnabled(!dictationEnabled); } }}
|
||||
>
|
||||
<Checkbox checked={dictationEnabled} onChange={setDictationEnabled} ariaLabel={t('settings.voice.page.field.enableVoiceInputAria')} />
|
||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.enableVoiceInput')}</span>
|
||||
</div>
|
||||
|
||||
{dictationEnabled && (<>
|
||||
<div className="pb-1.5 pt-0.5">
|
||||
<div className="flex min-w-0 flex-col gap-1.5">
|
||||
<div className="flex items-center gap-1.5">
|
||||
@@ -845,7 +1173,7 @@ export const VoiceSettings: React.FC = () => {
|
||||
</TooltipTrigger>
|
||||
<TooltipContent sideOffset={8} className="max-w-xs">
|
||||
<ul className="space-y-1">
|
||||
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.sttBrowser')}</li>
|
||||
<li><strong>{t('settings.voice.page.provider.local')}</strong> {t('settings.voice.page.tooltip.sttLocal')}</li>
|
||||
<li><strong>{t('settings.voice.page.provider.server')}</strong> {t('settings.voice.page.tooltip.sttServer')}</li>
|
||||
</ul>
|
||||
</TooltipContent>
|
||||
@@ -855,53 +1183,34 @@ export const VoiceSettings: React.FC = () => {
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={sttProvider === 'browser'}
|
||||
onClick={() => setSttProvider('browser')}
|
||||
aria-pressed={sttProvider === 'local'}
|
||||
onClick={() => setSttProvider('local')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.provider.browser')}
|
||||
{t('settings.voice.page.provider.local')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={sttProvider === 'server'}
|
||||
onClick={() => setSttProvider('server')}
|
||||
aria-pressed={sttProvider === 'openai-compatible'}
|
||||
onClick={() => setSttProvider('openai-compatible')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.provider.server')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={sttProvider === 'wasm'}
|
||||
onClick={() => setSttProvider('wasm')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.provider.wasm')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{sttProvider === 'server' && (
|
||||
<div className="py-1.5 space-y-2">
|
||||
<div
|
||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
aria-pressed={sttTranscribeOnStop}
|
||||
onClick={() => setSttTranscribeOnStop(!sttTranscribeOnStop)}
|
||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setSttTranscribeOnStop(!sttTranscribeOnStop); } }}
|
||||
>
|
||||
<Checkbox checked={sttTranscribeOnStop} onChange={setSttTranscribeOnStop} ariaLabel={t('settings.voice.page.field.transcribeOnStopAria')} />
|
||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.transcribeOnStop')}</span>
|
||||
</div>
|
||||
{sttProvider === 'local' && (
|
||||
<div className="py-1.5">
|
||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.model')}</span>
|
||||
<LocalModelPicker selectedModelId={sttLocalModel} onSelect={setSttLocalModel} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!audioStreamService.isSupported() && (
|
||||
<p className="typography-meta text-[var(--status-error)]">
|
||||
{t('settings.voice.page.field.sttBrowserSupportError')}
|
||||
</p>
|
||||
)}
|
||||
{sttProvider === 'openai-compatible' && (
|
||||
<div className="py-1.5 space-y-2">
|
||||
<div>
|
||||
<span className={cn("typography-ui-label text-foreground", !sttServerUrl.trim() && "text-[var(--status-error)]")}>
|
||||
{t('settings.voice.page.field.serverUrl')}
|
||||
@@ -979,129 +1288,14 @@ export const VoiceSettings: React.FC = () => {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-8 py-0.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.silenceThreshold')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={-60} max={-20} step={1} value={sttSilenceThresholdDb} onChange={(e) => setSttSilenceThresholdDb(Number(e.target.value))} className={sliderClass} />}
|
||||
<span className="typography-ui-label text-foreground tabular-nums min-w-[3.5rem] text-right">
|
||||
{sttSilenceThresholdDb} dB
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-8 py-0.5">
|
||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.silenceHold')}</span>
|
||||
<div className="flex items-center gap-2 w-fit">
|
||||
{!isMobile && <input type="range" min={500} max={3000} step={100} value={sttSilenceHoldMs} onChange={(e) => setSttSilenceHoldMs(Number(e.target.value))} className={sliderClass} />}
|
||||
<NumberInput value={sttSilenceHoldMs} onValueChange={setSttSilenceHoldMs} min={500} max={3000} step={100} className="w-20 tabular-nums" />
|
||||
<span className="typography-meta text-muted-foreground">{t('settings.voice.page.field.millisecondsUnit')}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sttProvider === 'wasm' && (
|
||||
<div className="py-1.5 space-y-2">
|
||||
<div>
|
||||
<span className="typography-ui-label text-muted-foreground">
|
||||
{t('settings.voice.page.stt.wasmModel')}
|
||||
</span>
|
||||
<Select value={wasmSttModel} onValueChange={setWasmSttModel}>
|
||||
<SelectTrigger className="mt-0.5">
|
||||
<SelectValue placeholder={t('settings.voice.page.stt.wasmModel')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{WASM_MODELS.map((m) => (
|
||||
<SelectItem key={m.id} value={m.id}>
|
||||
<div className="flex flex-col">
|
||||
<span>{m.name}</span>
|
||||
<span className="typography-ui-compact text-muted-foreground">
|
||||
{m.size} · {m.languages}
|
||||
</span>
|
||||
</div>
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<p className="typography-ui-compact text-muted-foreground mt-0.5">
|
||||
{WASM_MODELS.find((m) => m.id === wasmSttModel)?.description}
|
||||
</p>
|
||||
</div>
|
||||
{/* Model status indicator */}
|
||||
<WasmModelStatusIndicator modelId={wasmSttModel} />
|
||||
</div>
|
||||
)}
|
||||
</>)}
|
||||
</section>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Playback & Summarization */}
|
||||
<div data-settings-item="voice.playback" className="mb-8">
|
||||
<div className="mb-1 px-1">
|
||||
<h3 className="typography-ui-header font-medium text-foreground">
|
||||
{t('settings.voice.page.section.playbackAndSummary')}
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
<section className="px-2 pb-2 pt-0 space-y-0">
|
||||
<div
|
||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
aria-pressed={showMessageTTSButtons}
|
||||
onClick={() => setShowMessageTTSButtons(!showMessageTTSButtons)}
|
||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setShowMessageTTSButtons(!showMessageTTSButtons); } }}
|
||||
>
|
||||
<Checkbox checked={showMessageTTSButtons} onChange={setShowMessageTTSButtons} ariaLabel={t('settings.voice.page.field.messageReadAloudButtonAria')} />
|
||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.messageReadAloudButton')}</span>
|
||||
</div>
|
||||
|
||||
<div className="pb-1.5 pt-0.5">
|
||||
<div className="flex min-w-0 flex-col gap-1.5">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="typography-ui-label text-foreground">
|
||||
{t('settings.voice.page.field.ttsInputMode')}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={ttsInputMode === 'sanitized'}
|
||||
onClick={() => setTtsInputMode('sanitized')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.field.ttsInputModeSanitized')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="chip"
|
||||
size="xs"
|
||||
aria-pressed={ttsInputMode === 'raw'}
|
||||
onClick={() => setTtsInputMode('raw')}
|
||||
className="!font-normal"
|
||||
>
|
||||
{t('settings.voice.page.field.ttsInputModeRaw')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
{voiceModeEnabled && isSupported && (
|
||||
<div className="mt-2 px-2">
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
{t('settings.voice.page.hint.shiftClickPrefix')}
|
||||
{' '}
|
||||
<kbd className="px-1 py-0.5 mx-0.5 rounded border border-[var(--interactive-border)] bg-background typography-mono text-[10px]">Shift</kbd>
|
||||
{' + '}
|
||||
<kbd className="px-1 py-0.5 mx-0.5 rounded border border-[var(--interactive-border)] bg-background typography-mono text-[10px]">Click</kbd>
|
||||
{' '}
|
||||
{t('settings.voice.page.hint.shiftClickSuffix')}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -1030,7 +1030,7 @@ export const SettingsView: React.FC<SettingsViewProps> = ({ onClose, forceMobile
|
||||
: <Icon name={iconName!} className="h-4 w-4 shrink-0" />}
|
||||
<span className="flex items-center gap-1.5 whitespace-nowrap overflow-hidden transition-opacity duration-150 opacity-100">
|
||||
<span className="typography-ui-label font-normal truncate">{getPageTitle(page.slug)}</span>
|
||||
{(page.slug === 'voice' || page.slug === 'tunnel') && (
|
||||
{page.slug === 'tunnel' && (
|
||||
<span className="shrink-0 typography-micro px-1 rounded leading-none pb-px text-[var(--status-warning)] bg-[var(--status-warning)]/10">
|
||||
{t('settings.view.badge.beta')}
|
||||
</span>
|
||||
|
||||
@@ -1,392 +0,0 @@
|
||||
/**
|
||||
* BrowserVoiceButton Component
|
||||
*
|
||||
* Voice toggle button for browser-based voice chat with language selection.
|
||||
* Shows visual state indicators for different voice modes.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <BrowserVoiceButton />
|
||||
* ```
|
||||
*/
|
||||
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { isVSCodeRuntime } from '@/lib/desktop';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipProvider,
|
||||
TooltipTrigger,
|
||||
} from '@/components/ui/tooltip';
|
||||
import { VoiceStatusIndicator } from './VoiceStatusIndicator';
|
||||
import { toast } from '@/components/ui/toast';
|
||||
import { Icon } from "@/components/icon/Icon";
|
||||
import { useI18n } from '@/lib/i18n';
|
||||
|
||||
// Status text for accessibility and labels
|
||||
const statusLabels: Record<string, string> = {
|
||||
idle: 'Start Voice',
|
||||
listening: 'Listening',
|
||||
processing: 'Processing',
|
||||
speaking: 'AI Speaking',
|
||||
error: 'Voice Error',
|
||||
};
|
||||
|
||||
// iOS Safari detection utility
|
||||
const isIOSSafari = (): boolean => {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
const isIOS = /iphone|ipad|ipod/i.test(userAgent);
|
||||
const isSafari = /safari/i.test(userAgent) && !/chrome|crios|crmo/i.test(userAgent);
|
||||
return isIOS && isSafari;
|
||||
};
|
||||
|
||||
const normalizeVoiceErrorMessage = (error: string): string => {
|
||||
const isMediaDevicesError =
|
||||
error.includes('getUserMedia') ||
|
||||
error.includes('mediaDevices') ||
|
||||
error.includes('Cannot read properties of undefined');
|
||||
|
||||
if (!isMediaDevicesError) {
|
||||
return error;
|
||||
}
|
||||
|
||||
if (typeof window !== 'undefined' && !window.isSecureContext) {
|
||||
return 'Voice requires a secure connection (HTTPS) or localhost. Please use HTTPS or access via localhost.';
|
||||
}
|
||||
|
||||
return 'Microphone access is unavailable in this runtime. On desktop, check System Settings -> Privacy & Security -> Microphone for OpenChamber.';
|
||||
};
|
||||
|
||||
/**
|
||||
* Browser Voice Button with language selection
|
||||
*/
|
||||
export function BrowserVoiceButton() {
|
||||
const { t } = useI18n();
|
||||
const voiceModeEnabled = useConfigStore((s) => s.voiceModeEnabled);
|
||||
const sttProvider = useConfigStore((s) => s.sttProvider);
|
||||
const sttTranscribeOnStop = useConfigStore((s) => s.sttTranscribeOnStop);
|
||||
|
||||
const {
|
||||
status,
|
||||
isSupported,
|
||||
error,
|
||||
|
||||
startVoice,
|
||||
stopVoice,
|
||||
finishVoiceInput,
|
||||
conversationMode,
|
||||
toggleConversationMode,
|
||||
isMobile,
|
||||
} = useBrowserVoice();
|
||||
|
||||
const [isPressing, setIsPressing] = useState(false);
|
||||
const isVSCode = isVSCodeRuntime();
|
||||
const buttonSizeClass = isMobile ? 'h-8 w-8 min-h-[32px] min-w-[32px]' : (isVSCode ? 'h-5 w-5' : 'h-6 w-6');
|
||||
const iconSizeClass = isMobile ? 'h-[18px] w-[18px]' : (isVSCode ? 'h-4 w-4' : 'h-[18px] w-[18px]');
|
||||
const continuousIconSizeClass = 'size-[18px]';
|
||||
const clearHoverBackgroundClass = 'bg-transparent hover:!bg-transparent active:!bg-transparent focus-visible:!bg-transparent';
|
||||
|
||||
// Refs for touch handling
|
||||
const touchHandledRef = useRef(false);
|
||||
const isIOSSafariRef = useRef(false);
|
||||
const longPressTimerRef = useRef<NodeJS.Timeout | null>(null);
|
||||
const longPressTriggeredRef = useRef(false);
|
||||
const lastToastedErrorRef = useRef<string | null>(null);
|
||||
|
||||
// Initialize iOS detection on mount
|
||||
useEffect(() => {
|
||||
isIOSSafariRef.current = isIOSSafari();
|
||||
}, []);
|
||||
|
||||
// NOTE: Do NOT pre-request microphone permission on mount.
|
||||
// Permission is requested when the user explicitly taps the mic button.
|
||||
// Pre-requesting causes an unwanted permission prompt on mobile page load.
|
||||
|
||||
// Determine active states
|
||||
const isActive = status === 'listening' || status === 'speaking' || status === 'processing';
|
||||
const isError = status === 'error';
|
||||
const isIdle = status === 'idle';
|
||||
|
||||
const isSpeaking = status === 'speaking';
|
||||
// WASM STT always needs finishVoiceInput to flush the recorder and transcribe.
|
||||
// Server STT uses it when sttTranscribeOnStop is enabled.
|
||||
const canTranscribeOnStop = sttProvider === 'wasm' || (sttProvider === 'server' && sttTranscribeOnStop);
|
||||
const isListeningWithTranscribeOnStop = status === 'listening' && canTranscribeOnStop;
|
||||
|
||||
// Show toast notification when voice error occurs
|
||||
useEffect(() => {
|
||||
if (isError && error) {
|
||||
if (lastToastedErrorRef.current === error) {
|
||||
return;
|
||||
}
|
||||
lastToastedErrorRef.current = error;
|
||||
const displayError = normalizeVoiceErrorMessage(error);
|
||||
|
||||
toast.error(displayError, {
|
||||
duration: 5000,
|
||||
});
|
||||
}
|
||||
|
||||
if (!isError) {
|
||||
lastToastedErrorRef.current = null;
|
||||
}
|
||||
}, [isError, error]);
|
||||
|
||||
// Status text for accessibility
|
||||
const statusText = isError
|
||||
? error || 'Voice Error'
|
||||
: isListeningWithTranscribeOnStop
|
||||
? t('voice.action.finishAndTranscribe')
|
||||
: conversationMode && status === 'idle'
|
||||
? 'Start Voice (Continuous mode on)'
|
||||
: statusLabels[status] || 'Start Voice';
|
||||
|
||||
// Tooltip content based on state
|
||||
const getTooltipContent = () => {
|
||||
if (isError && error) {
|
||||
return normalizeVoiceErrorMessage(error);
|
||||
}
|
||||
if (isListeningWithTranscribeOnStop) {
|
||||
return t('voice.action.finishAndTranscribe');
|
||||
}
|
||||
if (isActive) {
|
||||
return 'Stop voice conversation';
|
||||
}
|
||||
if (isMobile) {
|
||||
return 'Start voice conversation';
|
||||
}
|
||||
return `Start voice conversation (Shift+Click for continuous mode) • Cmd/Ctrl+Shift+V to toggle`;
|
||||
};
|
||||
|
||||
// Handle voice activation (used by both click and touch)
|
||||
const activateVoice = useCallback(async () => {
|
||||
if (isActive) {
|
||||
if (status === 'listening' && canTranscribeOnStop) {
|
||||
finishVoiceInput();
|
||||
return;
|
||||
}
|
||||
stopVoice();
|
||||
} else if (status !== 'error') {
|
||||
// On mobile, we must NOT do any async operations before calling startVoice()
|
||||
// because iOS Safari requires SpeechRecognition.start() to be called
|
||||
// synchronously within the user gesture handler
|
||||
if (isMobile) {
|
||||
// Start voice immediately - no await before this!
|
||||
// Audio unlock is now handled inside startVoice() for mobile
|
||||
startVoice();
|
||||
} else {
|
||||
// Desktop can use async path
|
||||
try {
|
||||
await startVoice();
|
||||
} catch (err) {
|
||||
console.error('Failed to start voice:', err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Reset from error state
|
||||
if (isMobile) {
|
||||
startVoice();
|
||||
} else {
|
||||
try {
|
||||
await startVoice();
|
||||
} catch (err) {
|
||||
console.error('Failed to start voice:', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [isActive, status, canTranscribeOnStop, finishVoiceInput, startVoice, stopVoice, isMobile]);
|
||||
|
||||
// Handle Shift+Click to toggle conversation mode
|
||||
const handleClick = useCallback(async (e: React.MouseEvent) => {
|
||||
// Prevent double-firing if touch already handled this
|
||||
if (touchHandledRef.current) {
|
||||
touchHandledRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// Shift+Click toggles conversation mode
|
||||
if (e.shiftKey) {
|
||||
toggleConversationMode();
|
||||
return;
|
||||
}
|
||||
|
||||
await activateVoice();
|
||||
}, [activateVoice, toggleConversationMode]);
|
||||
|
||||
// Handle touch start for mobile devices
|
||||
const handleTouchStart = useCallback((e: React.TouchEvent) => {
|
||||
// Prevent default to stop mouse event emulation
|
||||
e.preventDefault();
|
||||
|
||||
// Mark that touch handled this interaction
|
||||
touchHandledRef.current = true;
|
||||
longPressTriggeredRef.current = false;
|
||||
|
||||
// Immediate visual feedback
|
||||
setIsPressing(true);
|
||||
|
||||
// Set up long-press timer for toggling conversation mode (500ms)
|
||||
longPressTimerRef.current = setTimeout(() => {
|
||||
longPressTriggeredRef.current = true;
|
||||
toggleConversationMode();
|
||||
// Haptic feedback if available
|
||||
if (navigator.vibrate) {
|
||||
navigator.vibrate(50);
|
||||
}
|
||||
setIsPressing(false);
|
||||
}, 500);
|
||||
}, [toggleConversationMode]);
|
||||
|
||||
// Handle touch end
|
||||
const handleTouchEnd = useCallback(() => {
|
||||
// Clear long-press timer
|
||||
if (longPressTimerRef.current) {
|
||||
clearTimeout(longPressTimerRef.current);
|
||||
longPressTimerRef.current = null;
|
||||
}
|
||||
|
||||
// Only activate voice if long-press wasn't triggered
|
||||
if (!longPressTriggeredRef.current) {
|
||||
activateVoice();
|
||||
}
|
||||
|
||||
setIsPressing(false);
|
||||
}, [activateVoice]);
|
||||
|
||||
// Handle touch cancel
|
||||
const handleTouchCancel = useCallback(() => {
|
||||
if (longPressTimerRef.current) {
|
||||
clearTimeout(longPressTimerRef.current);
|
||||
longPressTimerRef.current = null;
|
||||
}
|
||||
setIsPressing(false);
|
||||
}, []);
|
||||
|
||||
const handleToggleConversationMode = useCallback((event: React.MouseEvent<HTMLButtonElement>) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
toggleConversationMode();
|
||||
}, [toggleConversationMode]);
|
||||
|
||||
// If voice mode is disabled, don't render anything
|
||||
if (!voiceModeEnabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// If not supported, show disabled button with tooltip
|
||||
if (!isSupported) {
|
||||
const supportDetails = browserVoiceService.getSupportDetails();
|
||||
const tooltipMessage = !supportDetails.secureContext
|
||||
? 'Voice requires HTTPS or localhost. Please use a secure connection.'
|
||||
: !supportDetails.recognition
|
||||
? 'Speech recognition not supported in this browser. Try Chrome, Edge, or Safari.'
|
||||
: !supportDetails.synthesis
|
||||
? 'Speech synthesis not supported in this browser.'
|
||||
: 'Voice not supported in this browser';
|
||||
|
||||
return (
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
disabled
|
||||
aria-label={tooltipMessage}
|
||||
className={`${buttonSizeClass} p-0 ${clearHoverBackgroundClass}`}
|
||||
>
|
||||
<Icon name="mic-off" className={`${iconSizeClass} opacity-50`} />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" align="center">
|
||||
<p className="max-w-[200px] text-center">{tooltipMessage}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={`flex items-center ${isMobile ? 'gap-1' : 'gap-1.5'}`}>
|
||||
{/* Status indicator with label - show when active, simplified on mobile */}
|
||||
{isActive && !isMobile && (
|
||||
<VoiceStatusIndicator
|
||||
status={status}
|
||||
showLabel
|
||||
size="sm"
|
||||
className="mr-1"
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Voice button with tooltip */}
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onClick={handleClick}
|
||||
onTouchStart={handleTouchStart}
|
||||
onTouchEnd={handleTouchEnd}
|
||||
onTouchCancel={handleTouchCancel}
|
||||
aria-label={statusText}
|
||||
className={`
|
||||
relative
|
||||
${buttonSizeClass}
|
||||
p-0
|
||||
${clearHoverBackgroundClass}
|
||||
touch-manipulation
|
||||
${isPressing ? 'scale-95 opacity-80' : ''}
|
||||
${conversationMode && isIdle && isMobile ? 'ring-1 ring-primary/50' : ''}
|
||||
`}
|
||||
style={{
|
||||
WebkitTapHighlightColor: 'transparent',
|
||||
touchAction: 'manipulation',
|
||||
}}
|
||||
>
|
||||
{isActive ? (
|
||||
isSpeaking ? (
|
||||
// Green speaker icon when AI is speaking
|
||||
<Icon name="volume-up" className={`${iconSizeClass} text-green-400 animate-pulse`} />
|
||||
) : (
|
||||
// Red stop icon for listening/processing (both mobile and desktop)
|
||||
<Icon name="stop-circle" className={`${iconSizeClass} text-[var(--status-error)]`} />
|
||||
)
|
||||
) : (
|
||||
<VoiceStatusIndicator
|
||||
status={isError ? 'idle' : status}
|
||||
size={isMobile || isVSCode ? 'sm' : 'md'}
|
||||
/>
|
||||
)}
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" align="center">
|
||||
<p className="max-w-[200px] text-center">{getTooltipContent()}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
|
||||
{/* Conversation mode toggle button */}
|
||||
{(status === 'idle' || status === 'error') && (
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onPointerDownCapture={(event) => event.stopPropagation()}
|
||||
onClick={handleToggleConversationMode}
|
||||
aria-label={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
|
||||
title={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
|
||||
className={
|
||||
`${buttonSizeClass} p-0 ${clearHoverBackgroundClass} ${conversationMode ? 'text-[var(--status-info)] hover:text-[var(--status-info)]' : 'text-muted-foreground hover:text-foreground'}`
|
||||
}
|
||||
>
|
||||
<Icon name="voice-recognition" className={continuousIconSizeClass} />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import React from 'react';
|
||||
import { useVoiceContext } from '@/hooks/useVoiceContext';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
const VoiceContextBridge = React.memo(function VoiceContextBridge() {
|
||||
useVoiceContext();
|
||||
return null;
|
||||
});
|
||||
|
||||
/**
|
||||
* Provider component that initializes voice context sync.
|
||||
* Wrap the app with this to enable voice session awareness.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <VoiceProvider>
|
||||
* <App />
|
||||
* </VoiceProvider>
|
||||
* ```
|
||||
*/
|
||||
export function VoiceProvider({ children }: { children: React.ReactNode }) {
|
||||
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
|
||||
|
||||
return (
|
||||
<>
|
||||
{voiceModeEnabled ? <VoiceContextBridge /> : null}
|
||||
{children}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
/**
|
||||
* VoiceStatusIndicator Component
|
||||
*
|
||||
* Reusable visual indicator for voice mode states with icons, animations,
|
||||
* and optional status text labels.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* // Basic usage - icon only
|
||||
* <VoiceStatusIndicator status="listening" />
|
||||
*
|
||||
* // With label
|
||||
* <VoiceStatusIndicator status="listening" showLabel />
|
||||
*
|
||||
* // Different size
|
||||
* <VoiceStatusIndicator status="processing" size="lg" />
|
||||
* ```
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import type { BrowserVoiceStatus } from '@/hooks/useBrowserVoice';
|
||||
import { useI18n } from '@/lib/i18n';
|
||||
import { Icon } from "@/components/icon/Icon";
|
||||
import type { IconName } from "@/components/icon/icons";
|
||||
|
||||
export interface VoiceStatusIndicatorProps {
|
||||
/** Current voice status */
|
||||
status: BrowserVoiceStatus;
|
||||
/** Show text label next to icon */
|
||||
showLabel?: boolean;
|
||||
/** Size of the indicator */
|
||||
size?: 'sm' | 'md' | 'lg';
|
||||
/** Optional className for styling */
|
||||
className?: string;
|
||||
/** Whether conversation mode is active (shows indicator dot when idle) */
|
||||
conversationMode?: boolean;
|
||||
}
|
||||
|
||||
const sizeClasses = {
|
||||
sm: {
|
||||
icon: 'w-4 h-4',
|
||||
container: 'gap-1.5',
|
||||
},
|
||||
md: {
|
||||
icon: 'w-5 h-5',
|
||||
container: 'gap-2',
|
||||
},
|
||||
lg: {
|
||||
icon: 'w-6 h-6',
|
||||
container: 'gap-2.5',
|
||||
},
|
||||
};
|
||||
|
||||
const statusConfig: Record<
|
||||
BrowserVoiceStatus,
|
||||
{
|
||||
icon: IconName;
|
||||
color: string;
|
||||
labelKey:
|
||||
| 'voice.status.idle'
|
||||
| 'voice.status.listening'
|
||||
| 'voice.status.processing'
|
||||
| 'voice.status.speaking'
|
||||
| 'voice.status.error';
|
||||
animation?: string;
|
||||
}
|
||||
> = {
|
||||
idle: {
|
||||
icon: "mic-off",
|
||||
color: 'text-muted-foreground',
|
||||
labelKey: 'voice.status.idle',
|
||||
},
|
||||
listening: {
|
||||
icon: "mic",
|
||||
color: 'text-primary',
|
||||
labelKey: 'voice.status.listening',
|
||||
animation: 'animate-pulse',
|
||||
},
|
||||
processing: {
|
||||
icon: "loader-4",
|
||||
color: 'text-primary',
|
||||
labelKey: 'voice.status.processing',
|
||||
animation: 'animate-spin',
|
||||
},
|
||||
speaking: {
|
||||
icon: "volume-up",
|
||||
color: 'text-green-500',
|
||||
labelKey: 'voice.status.speaking',
|
||||
},
|
||||
error: {
|
||||
icon: "alert",
|
||||
color: 'text-destructive',
|
||||
labelKey: 'voice.status.error',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* VoiceStatusIndicator - Visual indicator for voice mode states
|
||||
*/
|
||||
export function VoiceStatusIndicator({
|
||||
status,
|
||||
showLabel = false,
|
||||
size = 'md',
|
||||
className = '',
|
||||
conversationMode = false,
|
||||
}: VoiceStatusIndicatorProps) {
|
||||
const { t } = useI18n();
|
||||
const config = statusConfig[status];
|
||||
const statusIconName = config.icon;
|
||||
const sizeClass = sizeClasses[size];
|
||||
const containerClass = showLabel ? sizeClass.container : '';
|
||||
|
||||
return (
|
||||
<div className={`flex items-center ${containerClass} ${className}`}>
|
||||
<div className="relative">
|
||||
<Icon name={statusIconName}
|
||||
className={`
|
||||
${sizeClass.icon}
|
||||
${config.color}
|
||||
${config.animation || ''}
|
||||
`}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
{/* Conversation mode indicator dot - only when idle and conversation mode is on */}
|
||||
{conversationMode && status === 'idle' && (
|
||||
<span
|
||||
className="absolute -top-0.5 -right-0.5 w-2 h-2 bg-green-500 rounded-full"
|
||||
aria-label={t('voice.status.conversationModeActiveAria')}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
{showLabel && (
|
||||
<span className={`typography-meta ${config.color}`}>
|
||||
{t(config.labelKey)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
export { VoiceProvider } from './VoiceProvider';
|
||||
export { BrowserVoiceButton } from './BrowserVoiceButton';
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,408 @@
|
||||
/**
|
||||
* Streaming dictation state machine.
|
||||
*
|
||||
* Status flow: idle -> recording -> uploading -> idle | failed.
|
||||
* While recording, mic PCM chunks stream to the server, which sends back live
|
||||
* partial transcripts. Confirm finalizes and resolves the full text; failed
|
||||
* dictations retain their audio segments so retry can replay them.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
|
||||
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
|
||||
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
|
||||
|
||||
export interface UseDictationOptions {
|
||||
onTranscript?: (text: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
canStart?: () => boolean;
|
||||
}
|
||||
|
||||
export interface UseDictationResult {
|
||||
status: DictationStatus;
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
partialTranscript: string;
|
||||
volume: number;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
errorReason: string | null;
|
||||
startDictation: () => Promise<void>;
|
||||
confirmDictation: () => Promise<string | null>;
|
||||
cancelDictation: () => Promise<void>;
|
||||
retryFailedDictation: () => Promise<string | null>;
|
||||
acceptPartialTranscript: () => string | null;
|
||||
discardFailedDictation: () => void;
|
||||
}
|
||||
|
||||
const DURATION_TICK_MS = 1000;
|
||||
|
||||
const toError = (value: unknown): Error =>
|
||||
value instanceof Error ? value : new Error(String(value));
|
||||
|
||||
const getDictationStartOptions = (): DictationStartOptions => {
|
||||
const state = useConfigStore.getState();
|
||||
const language = state.sttLanguage?.trim();
|
||||
if (state.sttProvider === 'openai-compatible') {
|
||||
return {
|
||||
provider: 'openai-compatible',
|
||||
...(language ? { language } : {}),
|
||||
openaiCompatible: {
|
||||
baseUrl: state.sttServerUrl,
|
||||
model: state.sttModel,
|
||||
...(state.sttApiKey ? { apiKey: state.sttApiKey } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
provider: 'local',
|
||||
...(language ? { language } : {}),
|
||||
localModel: state.sttLocalModel,
|
||||
};
|
||||
};
|
||||
|
||||
export function useDictation(options: UseDictationOptions = {}): UseDictationResult {
|
||||
const { onTranscript, onError, canStart } = options;
|
||||
|
||||
const [status, setStatus] = useState<DictationStatus>('idle');
|
||||
const [partialTranscript, setPartialTranscript] = useState('');
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [errorReason, setErrorReason] = useState<string | null>(null);
|
||||
|
||||
const statusRef = useRef(status);
|
||||
useEffect(() => {
|
||||
statusRef.current = status;
|
||||
}, [status]);
|
||||
|
||||
const latestPartialRef = useRef('');
|
||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const actionGateRef = useRef({ starting: false, confirming: false, cancelling: false });
|
||||
|
||||
const onTranscriptRef = useRef(onTranscript);
|
||||
const onErrorRef = useRef(onError);
|
||||
useEffect(() => {
|
||||
onTranscriptRef.current = onTranscript;
|
||||
onErrorRef.current = onError;
|
||||
}, [onTranscript, onError]);
|
||||
|
||||
const senderRef = useRef<DictationStreamSender | null>(null);
|
||||
if (!senderRef.current) {
|
||||
senderRef.current = new DictationStreamSender({
|
||||
client: dictationClient,
|
||||
getStartOptions: getDictationStartOptions,
|
||||
});
|
||||
}
|
||||
|
||||
const stopDurationTracking = useCallback(() => {
|
||||
if (durationIntervalRef.current) {
|
||||
clearInterval(durationIntervalRef.current);
|
||||
durationIntervalRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const startDurationTracking = useCallback(() => {
|
||||
if (durationIntervalRef.current) {
|
||||
return;
|
||||
}
|
||||
durationIntervalRef.current = setInterval(() => {
|
||||
setDuration((prev) => prev + 1);
|
||||
}, DURATION_TICK_MS);
|
||||
}, []);
|
||||
|
||||
const reportError = useCallback((err: unknown) => {
|
||||
const normalized = toError(err);
|
||||
setError(normalized.message);
|
||||
const reason = (normalized as Error & { reasonCode?: string }).reasonCode;
|
||||
setErrorReason(typeof reason === 'string' ? reason : null);
|
||||
onErrorRef.current?.(normalized);
|
||||
}, []);
|
||||
|
||||
const clearError = useCallback(() => {
|
||||
setError(null);
|
||||
setErrorReason(null);
|
||||
}, []);
|
||||
|
||||
const clearStreamingState = useCallback(() => {
|
||||
senderRef.current?.clearAll();
|
||||
latestPartialRef.current = '';
|
||||
setPartialTranscript('');
|
||||
}, []);
|
||||
|
||||
// Live partial transcripts for the active dictation.
|
||||
useEffect(() => {
|
||||
return dictationClient.onPartial((dictationId, text) => {
|
||||
const activeDictationId = senderRef.current?.getDictationId();
|
||||
if (!activeDictationId || dictationId !== activeDictationId) {
|
||||
return;
|
||||
}
|
||||
latestPartialRef.current = text;
|
||||
setPartialTranscript(text);
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Restart the stream (replaying buffered segments) after a reconnect.
|
||||
useEffect(() => {
|
||||
return dictationClient.subscribeConnectionStatus((connected) => {
|
||||
if (!connected) {
|
||||
return;
|
||||
}
|
||||
if (statusRef.current !== 'recording') {
|
||||
return;
|
||||
}
|
||||
void senderRef.current?.restartStream().catch((err) => {
|
||||
reportError(err);
|
||||
});
|
||||
});
|
||||
}, [reportError]);
|
||||
|
||||
const audio = useDictationAudioSource({
|
||||
onPcmSegment: (audioData) => {
|
||||
senderRef.current?.enqueueSegment(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
onErrorRef.current?.(err);
|
||||
},
|
||||
});
|
||||
const audioStopRef = useRef(audio.stop);
|
||||
useEffect(() => {
|
||||
audioStopRef.current = audio.stop;
|
||||
}, [audio.stop]);
|
||||
|
||||
const handleSuccess = useCallback(
|
||||
(text: string): string | null => {
|
||||
setDuration(0);
|
||||
setStatus('idle');
|
||||
const transcriptText = text.trim().length > 0 ? text.trim() : latestPartialRef.current.trim();
|
||||
clearStreamingState();
|
||||
if (!transcriptText) {
|
||||
return null;
|
||||
}
|
||||
onTranscriptRef.current?.(transcriptText);
|
||||
return transcriptText;
|
||||
},
|
||||
[clearStreamingState],
|
||||
);
|
||||
|
||||
const handleFailure = useCallback(
|
||||
(failure: unknown) => {
|
||||
if (senderRef.current?.hasSegments()) {
|
||||
setStatus('failed');
|
||||
} else {
|
||||
setStatus('idle');
|
||||
}
|
||||
reportError(failure);
|
||||
},
|
||||
[reportError],
|
||||
);
|
||||
|
||||
const startDictation = useCallback(async () => {
|
||||
const gate = actionGateRef.current;
|
||||
if (gate.starting || gate.confirming || gate.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (statusRef.current !== 'idle') {
|
||||
return;
|
||||
}
|
||||
if (canStart && !canStart()) {
|
||||
return;
|
||||
}
|
||||
|
||||
gate.starting = true;
|
||||
clearError();
|
||||
setPartialTranscript('');
|
||||
setDuration(0);
|
||||
setStatus('recording');
|
||||
statusRef.current = 'recording';
|
||||
clearStreamingState();
|
||||
|
||||
try {
|
||||
await audio.start();
|
||||
startDurationTracking();
|
||||
// Open the stream eagerly so partials start flowing immediately.
|
||||
await senderRef.current?.restartStream().catch((err) => {
|
||||
// Non-fatal: segments buffer locally and finish() retries the
|
||||
// start, but surface the reason (e.g. model downloading) so
|
||||
// the overlay can show it.
|
||||
reportError(err);
|
||||
});
|
||||
} catch (err) {
|
||||
await audio.stop().catch(() => undefined);
|
||||
stopDurationTracking();
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
reportError(err);
|
||||
} finally {
|
||||
gate.starting = false;
|
||||
}
|
||||
}, [audio, canStart, clearError, clearStreamingState, reportError, startDurationTracking, stopDurationTracking]);
|
||||
|
||||
const cancelDictation = useCallback(async () => {
|
||||
const gate = actionGateRef.current;
|
||||
if (gate.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (statusRef.current !== 'recording' && statusRef.current !== 'uploading') {
|
||||
return;
|
||||
}
|
||||
gate.cancelling = true;
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
clearError();
|
||||
|
||||
// Optimistic: dismiss the overlay immediately. Tearing down the audio
|
||||
// graph (AudioContext.close) can take up to ~1s on mobile WebViews and
|
||||
// must not delay the visible response to Cancel.
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
try {
|
||||
senderRef.current?.cancel();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
clearStreamingState();
|
||||
|
||||
try {
|
||||
await audio.stop();
|
||||
} catch {
|
||||
// Cancelled anyway; mic teardown failures are not user-actionable.
|
||||
} finally {
|
||||
gate.cancelling = false;
|
||||
}
|
||||
}, [audio, clearError, clearStreamingState, stopDurationTracking]);
|
||||
|
||||
const confirmDictation = useCallback(async (): Promise<string | null> => {
|
||||
const gate = actionGateRef.current;
|
||||
if (gate.confirming) {
|
||||
return null;
|
||||
}
|
||||
if (statusRef.current !== 'recording') {
|
||||
return null;
|
||||
}
|
||||
|
||||
gate.confirming = true;
|
||||
clearError();
|
||||
stopDurationTracking();
|
||||
|
||||
try {
|
||||
await audio.stop();
|
||||
setStatus('uploading');
|
||||
statusRef.current = 'uploading';
|
||||
|
||||
const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
|
||||
if (finalSeq < 0) {
|
||||
return handleSuccess('');
|
||||
}
|
||||
|
||||
const result = await senderRef.current!.finish(finalSeq);
|
||||
return handleSuccess(result.text);
|
||||
} catch (err) {
|
||||
handleFailure(err);
|
||||
return null;
|
||||
} finally {
|
||||
gate.confirming = false;
|
||||
}
|
||||
}, [audio, clearError, handleFailure, handleSuccess, stopDurationTracking]);
|
||||
|
||||
const retryFailedDictation = useCallback(async (): Promise<string | null> => {
|
||||
if (statusRef.current !== 'failed' || !senderRef.current?.hasSegments()) {
|
||||
return null;
|
||||
}
|
||||
clearError();
|
||||
setStatus('uploading');
|
||||
statusRef.current = 'uploading';
|
||||
|
||||
try {
|
||||
senderRef.current.resetStreamForReplay();
|
||||
const finalSeq = senderRef.current.getFinalSeq();
|
||||
const result = await senderRef.current.finish(finalSeq);
|
||||
return handleSuccess(result.text);
|
||||
} catch (err) {
|
||||
handleFailure(err);
|
||||
return null;
|
||||
}
|
||||
}, [clearError, handleFailure, handleSuccess]);
|
||||
|
||||
/**
|
||||
* Failed dictations still hold the last streamed partial transcript.
|
||||
* Accept it as-is instead of retrying the full transcription.
|
||||
*/
|
||||
const acceptPartialTranscript = useCallback((): string | null => {
|
||||
if (statusRef.current !== 'failed') {
|
||||
return null;
|
||||
}
|
||||
const text = latestPartialRef.current.trim();
|
||||
setDuration(0);
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
clearError();
|
||||
try {
|
||||
senderRef.current?.cancel();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
clearStreamingState();
|
||||
if (!text) {
|
||||
return null;
|
||||
}
|
||||
onTranscriptRef.current?.(text);
|
||||
return text;
|
||||
}, [clearError, clearStreamingState]);
|
||||
|
||||
const discardFailedDictation = useCallback(() => {
|
||||
setDuration(0);
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
clearError();
|
||||
clearStreamingState();
|
||||
}, [clearError, clearStreamingState]);
|
||||
|
||||
// While recording without an open stream (e.g. the model is still
|
||||
// downloading), retry the stream start so live partials kick in as soon
|
||||
// as the provider becomes ready. Buffered segments replay on success.
|
||||
useEffect(() => {
|
||||
if (status !== 'recording' || errorReason !== 'model_download_in_progress') {
|
||||
return;
|
||||
}
|
||||
const interval = setInterval(() => {
|
||||
if (statusRef.current !== 'recording' || senderRef.current?.getDictationId()) {
|
||||
return;
|
||||
}
|
||||
senderRef.current?.restartStream().then(() => {
|
||||
clearError();
|
||||
}).catch((err) => {
|
||||
reportError(err);
|
||||
});
|
||||
}, 3000);
|
||||
return () => clearInterval(interval);
|
||||
}, [status, errorReason, clearError, reportError]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stopDurationTracking();
|
||||
void audioStopRef.current().catch(() => undefined);
|
||||
senderRef.current?.cancel();
|
||||
};
|
||||
}, [stopDurationTracking]);
|
||||
|
||||
return {
|
||||
status,
|
||||
isRecording: status === 'recording',
|
||||
isProcessing: status === 'uploading',
|
||||
partialTranscript,
|
||||
volume: audio.volume,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
startDictation,
|
||||
confirmDictation,
|
||||
cancelDictation,
|
||||
retryFailedDictation,
|
||||
acceptPartialTranscript,
|
||||
discardFailedDictation,
|
||||
};
|
||||
}
|
||||
@@ -474,6 +474,18 @@ export const useKeyboardShortcuts = () => {
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventMatchesShortcut(e, combo('toggle_dictation'))) {
|
||||
const { activeMainTab, isCommandPaletteOpen, isHelpDialogOpen, isSessionSwitcherOpen, isSettingsDialogOpen } = useUIStore.getState();
|
||||
if (activeMainTab !== 'chat' || isCommandPaletteOpen || isHelpDialogOpen || isSessionSwitcherOpen || isSettingsDialogOpen) {
|
||||
return;
|
||||
}
|
||||
e.preventDefault();
|
||||
// Dictation state lives inside the composer's isolated component;
|
||||
// toggle it via an event instead of subscribing this hot hook to it.
|
||||
window.dispatchEvent(new CustomEvent('openchamber:dictation-toggle'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.key === 'Escape') {
|
||||
const target = e.target as Element | null;
|
||||
const isInsideDialog = Boolean(target?.closest('[role="dialog"]'));
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* useLocalTTS Hook
|
||||
*
|
||||
* React hook for local (Kokoro via sherpa-onnx) text-to-speech playback.
|
||||
* Synthesis runs on the OpenChamber server in the dictation worker.
|
||||
*
|
||||
* Long texts are pipelined by sentence chunks: the first chunk starts playing
|
||||
* as soon as it is synthesized while the next chunk synthesizes in the
|
||||
* background, so time-to-first-audio stays ~1 chunk regardless of message
|
||||
* length.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
|
||||
export interface LocalTTSSpeakOptions {
|
||||
/** Kokoro speaker id (0-10) */
|
||||
speakerId?: number;
|
||||
/** Playback speed multiplier (1.0 = normal) */
|
||||
speed?: number;
|
||||
onStart?: () => void;
|
||||
onEnd?: () => void;
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
export interface UseLocalTTSReturn {
|
||||
isPlaying: boolean;
|
||||
error: string | null;
|
||||
speak: (text: string, options?: LocalTTSSpeakOptions) => Promise<void>;
|
||||
stop: () => void;
|
||||
/** Unlock audio for mobile Safari - call this on user gesture */
|
||||
unlockAudio: () => Promise<void>;
|
||||
}
|
||||
|
||||
/** Target chunk size: big enough to amortize requests, small enough for low latency. */
|
||||
const MIN_CHUNK_CHARS = 60;
|
||||
const MAX_CHUNK_CHARS = 400;
|
||||
|
||||
/**
|
||||
* Split text into sentence-aligned chunks for pipelined synthesis.
|
||||
* Sentences are merged until MIN_CHUNK_CHARS and hard-split at
|
||||
* MAX_CHUNK_CHARS so a single run-on sentence cannot stall the pipeline.
|
||||
*/
|
||||
export function splitTextForSynthesis(text: string): string[] {
|
||||
const normalized = text.replace(/\s+/g, ' ').trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const sentences = normalized.match(/[^.!?…]+[.!?…]+["')\]]*\s*|[^.!?…]+$/g) ?? [normalized];
|
||||
|
||||
const chunks: string[] = [];
|
||||
let current = '';
|
||||
for (const sentence of sentences) {
|
||||
for (let offset = 0; offset < sentence.length; offset += MAX_CHUNK_CHARS) {
|
||||
const piece = sentence.slice(offset, offset + MAX_CHUNK_CHARS);
|
||||
if (current && current.length + piece.length > MAX_CHUNK_CHARS) {
|
||||
chunks.push(current.trim());
|
||||
current = '';
|
||||
}
|
||||
current += piece;
|
||||
if (current.length >= MIN_CHUNK_CHARS && /[.!?…]["')\]]*\s*$/.test(current)) {
|
||||
chunks.push(current.trim());
|
||||
current = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
if (current.trim()) {
|
||||
chunks.push(current.trim());
|
||||
}
|
||||
|
||||
// Latency trim for the FIRST chunk only: if the opening sentence is long,
|
||||
// split it at a clause boundary (comma/semicolon/colon) so the first
|
||||
// audio starts sooner. Only the first chunk gets this treatment — comma
|
||||
// splits inside every sentence would chop the prosody, and after playback
|
||||
// starts the pipeline hides synthesis time anyway.
|
||||
if (chunks.length > 0 && chunks[0].length > 120) {
|
||||
const first = chunks[0];
|
||||
const clauseBreak = /[,;:]\s/g;
|
||||
let bestBreak = -1;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = clauseBreak.exec(first)) !== null) {
|
||||
const end = match.index + 1;
|
||||
if (end < 40) {
|
||||
continue;
|
||||
}
|
||||
if (end > 160) {
|
||||
break;
|
||||
}
|
||||
bestBreak = end;
|
||||
break;
|
||||
}
|
||||
if (bestBreak > 0) {
|
||||
const head = first.slice(0, bestBreak).trim();
|
||||
const tail = first.slice(bestBreak).trim();
|
||||
chunks.splice(0, 1, head, tail);
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
let sharedAudioContext: AudioContext | null = null;
|
||||
|
||||
function getAudioContext(): AudioContext {
|
||||
if (!sharedAudioContext) {
|
||||
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
return sharedAudioContext;
|
||||
}
|
||||
|
||||
interface PlaybackSession {
|
||||
cancelled: boolean;
|
||||
abort: AbortController;
|
||||
}
|
||||
|
||||
export function useLocalTTS(): UseLocalTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const sessionRef = useRef<PlaybackSession | null>(null);
|
||||
|
||||
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||
try {
|
||||
const ctx = getAudioContext();
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
}
|
||||
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(ctx.destination);
|
||||
source.start(0);
|
||||
} catch {
|
||||
// Unlocking is best-effort.
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
const session = sessionRef.current;
|
||||
if (session) {
|
||||
session.cancelled = true;
|
||||
session.abort.abort();
|
||||
sessionRef.current = null;
|
||||
}
|
||||
if (audioSourceRef.current) {
|
||||
try {
|
||||
audioSourceRef.current.onended = null;
|
||||
audioSourceRef.current.stop();
|
||||
} catch {
|
||||
// Already stopped
|
||||
}
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
setIsPlaying(false);
|
||||
}, []);
|
||||
|
||||
const speak = useCallback(async (text: string, options?: LocalTTSSpeakOptions): Promise<void> => {
|
||||
stop();
|
||||
|
||||
const chunks = splitTextForSynthesis(text);
|
||||
if (chunks.length === 0) {
|
||||
setError('No text to speak');
|
||||
options?.onError?.('No text to speak');
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
|
||||
const session: PlaybackSession = { cancelled: false, abort: new AbortController() };
|
||||
sessionRef.current = session;
|
||||
|
||||
const fetchChunk = async (chunk: string): Promise<ArrayBuffer> => {
|
||||
const response = await runtimeFetch('/api/dictation/tts/speak', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
text: chunk,
|
||||
...(typeof options?.speakerId === 'number' ? { speakerId: options.speakerId } : {}),
|
||||
...(typeof options?.speed === 'number' ? { speed: options.speed } : {}),
|
||||
}),
|
||||
signal: session.abort.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
return (await response.blob()).arrayBuffer();
|
||||
};
|
||||
|
||||
const playBuffer = async (arrayBuffer: ArrayBuffer): Promise<void> => {
|
||||
const ctx = getAudioContext();
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
}
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
if (session.cancelled) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
audioSourceRef.current = source;
|
||||
source.onended = () => {
|
||||
if (audioSourceRef.current === source) {
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
source.start(0);
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
setIsPlaying(true);
|
||||
options?.onStart?.();
|
||||
|
||||
// Pipelined: synthesize chunk N+1 while chunk N is playing.
|
||||
let nextFetch: Promise<ArrayBuffer> = fetchChunk(chunks[0]);
|
||||
for (let i = 0; i < chunks.length; i += 1) {
|
||||
const buffer = await nextFetch;
|
||||
if (session.cancelled) {
|
||||
return;
|
||||
}
|
||||
if (i + 1 < chunks.length) {
|
||||
nextFetch = fetchChunk(chunks[i + 1]);
|
||||
}
|
||||
await playBuffer(buffer);
|
||||
if (session.cancelled) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
options?.onEnd?.();
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError' || session.cancelled) {
|
||||
return;
|
||||
}
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||
setError(errorMsg);
|
||||
options?.onError?.(errorMsg);
|
||||
setIsPlaying(false);
|
||||
} finally {
|
||||
if (sessionRef.current === session) {
|
||||
sessionRef.current = null;
|
||||
}
|
||||
}
|
||||
}, [stop]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
error,
|
||||
speak,
|
||||
stop,
|
||||
unlockAudio,
|
||||
};
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { useCallback, useState } from 'react';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useServerTTS } from './useServerTTS';
|
||||
import { useSayTTS } from './useSayTTS';
|
||||
import { useLocalTTS } from './useLocalTTS';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { sanitizeForTTS } from '@/lib/voice/summarize';
|
||||
|
||||
@@ -29,6 +30,7 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
const speechPitch = useConfigStore((state) => state.speechPitch);
|
||||
const speechVolume = useConfigStore((state) => state.speechVolume);
|
||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
|
||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
|
||||
@@ -48,13 +50,15 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS({
|
||||
enabled: shouldCheckSayAvailability,
|
||||
});
|
||||
const { speak: speakLocalTTS, stop: stopLocalTTS } = useLocalTTS();
|
||||
|
||||
const stop = useCallback(() => {
|
||||
setIsPlaying(false);
|
||||
stopServerTTS();
|
||||
stopSayTTS();
|
||||
stopLocalTTS();
|
||||
browserVoiceService.cancelSpeech();
|
||||
}, [stopServerTTS, stopSayTTS]);
|
||||
}, [stopServerTTS, stopSayTTS, stopLocalTTS]);
|
||||
|
||||
const play = useCallback(async (text: string) => {
|
||||
if (!text.trim()) return;
|
||||
@@ -84,6 +88,13 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else if (voiceProvider === 'local') {
|
||||
await speakLocalTTS(sanitizedText, {
|
||||
speakerId: localTtsVoiceId,
|
||||
speed: speechRate,
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||
await speakSayTTS(sanitizedText, {
|
||||
@@ -129,6 +140,8 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
ttsInputMode,
|
||||
speakServerTTS,
|
||||
speakSayTTS,
|
||||
speakLocalTTS,
|
||||
localTtsVoiceId,
|
||||
stop,
|
||||
]);
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useSessionUIStore } from '@/sync/session-ui-store';
|
||||
import { useSessionPermissions, useSessionTextMessages } from '@/sync/sync-context';
|
||||
import { voiceHooks, isVoiceSessionStarted } from '@/lib/voice';
|
||||
|
||||
/**
|
||||
* Hook that syncs session events (messages, permissions) to the voice agent.
|
||||
* Call this inside VoiceProvider to enable session awareness during voice.
|
||||
*/
|
||||
export function useVoiceContext() {
|
||||
const currentSessionId = useSessionUIStore((s) => s.currentSessionId);
|
||||
const messages = useSessionTextMessages(currentSessionId ?? '');
|
||||
const permissions = useSessionPermissions(currentSessionId ?? '');
|
||||
|
||||
// Track last seen message count to only forward new messages
|
||||
const lastMessageCountRef = useRef(0);
|
||||
|
||||
// Forward new messages to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !messages || messages.length === 0 || !isVoiceSessionStarted()) return;
|
||||
|
||||
const currentCount = messages.length;
|
||||
if (currentCount <= lastMessageCountRef.current) return;
|
||||
|
||||
// Get only new messages (messages since last check)
|
||||
const newMessages = messages.slice(lastMessageCountRef.current);
|
||||
lastMessageCountRef.current = currentCount;
|
||||
|
||||
const formattedMessages = newMessages.map(m => ({
|
||||
role: m.role ?? '',
|
||||
content: m.text,
|
||||
}));
|
||||
|
||||
voiceHooks.onMessages(currentSessionId, formattedMessages);
|
||||
}, [currentSessionId, messages]);
|
||||
|
||||
// Forward permission requests to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !permissions || permissions.length === 0) return;
|
||||
if (!isVoiceSessionStarted()) return;
|
||||
|
||||
const request = permissions[0];
|
||||
if (!request) return;
|
||||
|
||||
voiceHooks.onPermissionRequested(
|
||||
currentSessionId,
|
||||
request.id,
|
||||
request.permission,
|
||||
request.metadata
|
||||
);
|
||||
}, [currentSessionId, permissions]);
|
||||
|
||||
// Reset message count when session changes
|
||||
useEffect(() => {
|
||||
lastMessageCountRef.current = 0;
|
||||
}, [currentSessionId]);
|
||||
}
|
||||
@@ -178,14 +178,12 @@ export type DesktopSettings = {
|
||||
responseStyleEnabled?: boolean;
|
||||
responseStylePreset?: 'concise' | 'detailed' | 'mentor' | 'pushback' | 'noFiller' | 'matchEnergy' | 'warmPeer' | 'custom';
|
||||
responseStyleCustomInstructions?: string;
|
||||
sttProvider?: 'browser' | 'server' | 'wasm';
|
||||
dictationEnabled?: boolean;
|
||||
sttProvider?: 'local' | 'openai-compatible';
|
||||
sttServerUrl?: string;
|
||||
sttModel?: string;
|
||||
wasmSttModel?: string;
|
||||
sttLocalModel?: string;
|
||||
sttLanguage?: string;
|
||||
sttSilenceThresholdDb?: number;
|
||||
sttSilenceHoldMs?: number;
|
||||
sttTranscribeOnStop?: boolean;
|
||||
// Global draft welcome starters (pinned commands/skills), persisted to settings.json
|
||||
draftStarters?: DraftStarterRef[];
|
||||
};
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
/**
|
||||
* WebSocket client for the OpenChamber dictation endpoint (/api/dictation/ws).
|
||||
*
|
||||
* One shared client per app. The socket is opened lazily when a dictation
|
||||
* starts and closed after an idle delay. URLs are resolved at connect time via
|
||||
* the runtime URL resolver so runtime switches never leak a stale endpoint.
|
||||
*/
|
||||
|
||||
import { getRuntimeUrlResolver } from '@/lib/runtime-url';
|
||||
import { refreshRuntimeUrlAuthToken } from '@/lib/runtime-auth';
|
||||
|
||||
export interface DictationStartOptions {
|
||||
provider?: 'local' | 'openai-compatible';
|
||||
language?: string;
|
||||
localModel?: string;
|
||||
openaiCompatible?: {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface DictationServerMessage {
|
||||
type: string;
|
||||
dictationId?: string;
|
||||
ackSeq?: number;
|
||||
text?: string;
|
||||
timeoutMs?: number;
|
||||
error?: string;
|
||||
retryable?: boolean;
|
||||
reasonCode?: string;
|
||||
}
|
||||
|
||||
interface DictationStreamError extends Error {
|
||||
retryable: boolean;
|
||||
reasonCode?: string;
|
||||
}
|
||||
|
||||
const createStreamError = (message: string, retryable: boolean, reasonCode?: string): DictationStreamError => {
|
||||
const error = new Error(message) as DictationStreamError;
|
||||
error.name = 'DictationStreamError';
|
||||
error.retryable = retryable;
|
||||
if (reasonCode) {
|
||||
error.reasonCode = reasonCode;
|
||||
}
|
||||
return error;
|
||||
};
|
||||
|
||||
const CONNECT_TIMEOUT_MS = 10000;
|
||||
const START_TIMEOUT_MS = 15000;
|
||||
const IDLE_CLOSE_DELAY_MS = 30000;
|
||||
const DEFAULT_FINISH_TIMEOUT_MS = 30000;
|
||||
|
||||
type ConnectionStatusListener = (connected: boolean) => void;
|
||||
type PartialListener = (dictationId: string, text: string) => void;
|
||||
|
||||
interface PendingStart {
|
||||
resolve: () => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
interface PendingFinish {
|
||||
resolve: (result: { text: string }) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: ReturnType<typeof setTimeout> | null;
|
||||
}
|
||||
|
||||
export class DictationClient {
|
||||
private socket: WebSocket | null = null;
|
||||
private connectPromise: Promise<void> | null = null;
|
||||
private idleCloseTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private readonly pendingStarts = new Map<string, PendingStart>();
|
||||
private readonly pendingFinishes = new Map<string, PendingFinish>();
|
||||
private readonly connectionListeners = new Set<ConnectionStatusListener>();
|
||||
private readonly partialListeners = new Set<PartialListener>();
|
||||
private activeDictations = 0;
|
||||
|
||||
get isConnected(): boolean {
|
||||
return this.socket?.readyState === WebSocket.OPEN;
|
||||
}
|
||||
|
||||
subscribeConnectionStatus(listener: ConnectionStatusListener): () => void {
|
||||
this.connectionListeners.add(listener);
|
||||
return () => {
|
||||
this.connectionListeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
onPartial(listener: PartialListener): () => void {
|
||||
this.partialListeners.add(listener);
|
||||
return () => {
|
||||
this.partialListeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
async ensureConnected(): Promise<void> {
|
||||
if (this.isConnected) {
|
||||
return;
|
||||
}
|
||||
if (this.connectPromise) {
|
||||
await this.connectPromise;
|
||||
return;
|
||||
}
|
||||
|
||||
// A WebSocket upgrade can't carry an Authorization header, so it
|
||||
// authenticates via the oc_url_token query param. Mint/await a valid
|
||||
// token BEFORE connecting — the sync getter returns "" while the token
|
||||
// is unminted or inside its expiry skew, and the server would reject
|
||||
// the upgrade with 401.
|
||||
try {
|
||||
await refreshRuntimeUrlAuthToken();
|
||||
} catch {
|
||||
// No auth configured (local runtime) — proceed without a token.
|
||||
}
|
||||
|
||||
this.connectPromise = new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let socket: WebSocket;
|
||||
try {
|
||||
const url = getRuntimeUrlResolver().websocket('/api/dictation/ws');
|
||||
socket = new WebSocket(url);
|
||||
} catch (error) {
|
||||
this.connectPromise = null;
|
||||
reject(error instanceof Error ? error : new Error(String(error)));
|
||||
return;
|
||||
}
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
this.connectPromise = null;
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
reject(new Error('Dictation connection timed out'));
|
||||
}
|
||||
}, CONNECT_TIMEOUT_MS);
|
||||
|
||||
socket.onopen = () => {
|
||||
// Wait for the server 'ready' frame before resolving.
|
||||
};
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
let message: DictationServerMessage;
|
||||
try {
|
||||
message = JSON.parse(String(event.data));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!settled && message.type === 'ready') {
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
this.socket = socket;
|
||||
this.connectPromise = null;
|
||||
this.notifyConnection(true);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
this.handleMessage(message);
|
||||
};
|
||||
|
||||
socket.onerror = () => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
this.connectPromise = null;
|
||||
reject(new Error('Dictation connection failed'));
|
||||
}
|
||||
};
|
||||
|
||||
socket.onclose = () => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
this.connectPromise = null;
|
||||
reject(new Error('Dictation connection closed'));
|
||||
return;
|
||||
}
|
||||
if (this.socket === socket) {
|
||||
this.socket = null;
|
||||
this.rejectAllPending(new Error('Dictation connection lost'));
|
||||
this.notifyConnection(false);
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
await this.connectPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a dictation stream. Resolves once the server acks the stream.
|
||||
*/
|
||||
async startDictationStream(
|
||||
dictationId: string,
|
||||
format: string,
|
||||
options: DictationStartOptions,
|
||||
): Promise<void> {
|
||||
await this.ensureConnected();
|
||||
this.activeDictations += 1;
|
||||
this.clearIdleCloseTimer();
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.pendingStarts.delete(dictationId);
|
||||
this.releaseDictation();
|
||||
reject(new Error('Dictation start timed out'));
|
||||
}, START_TIMEOUT_MS);
|
||||
|
||||
this.pendingStarts.set(dictationId, {
|
||||
resolve: () => {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
},
|
||||
reject: (error) => {
|
||||
clearTimeout(timeout);
|
||||
this.releaseDictation();
|
||||
reject(error);
|
||||
},
|
||||
timeout });
|
||||
|
||||
if (!this.send({ type: 'start', dictationId, format, options })) {
|
||||
clearTimeout(timeout);
|
||||
this.pendingStarts.delete(dictationId);
|
||||
this.releaseDictation();
|
||||
reject(new Error('Dictation connection lost'));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
sendDictationStreamChunk(dictationId: string, seq: number, audioBase64: string): boolean {
|
||||
return this.send({ type: 'chunk', dictationId, seq, audio: audioBase64 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Finish a dictation stream. Resolves with the final transcript.
|
||||
*/
|
||||
finishDictationStream(dictationId: string, finalSeq: number): Promise<{ text: string }> {
|
||||
return new Promise<{ text: string }>((resolve, reject) => {
|
||||
const pending: PendingFinish = {
|
||||
resolve: (result) => {
|
||||
if (pending.timeout) {
|
||||
clearTimeout(pending.timeout);
|
||||
}
|
||||
this.releaseDictation();
|
||||
resolve(result);
|
||||
},
|
||||
reject: (error) => {
|
||||
if (pending.timeout) {
|
||||
clearTimeout(pending.timeout);
|
||||
}
|
||||
this.releaseDictation();
|
||||
reject(error);
|
||||
},
|
||||
timeout: null,
|
||||
};
|
||||
pending.timeout = setTimeout(() => {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
this.releaseDictation();
|
||||
reject(new Error('Timed out waiting for transcription'));
|
||||
}, DEFAULT_FINISH_TIMEOUT_MS);
|
||||
|
||||
this.pendingFinishes.set(dictationId, pending);
|
||||
|
||||
if (!this.send({ type: 'finish', dictationId, finalSeq })) {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
pending.reject(new Error('Dictation connection lost'));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
cancelDictationStream(dictationId: string): void {
|
||||
this.send({ type: 'cancel', dictationId });
|
||||
const start = this.pendingStarts.get(dictationId);
|
||||
if (start) {
|
||||
this.pendingStarts.delete(dictationId);
|
||||
start.reject(new Error('Dictation cancelled'));
|
||||
}
|
||||
const finish = this.pendingFinishes.get(dictationId);
|
||||
if (finish) {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
finish.reject(new Error('Dictation cancelled'));
|
||||
}
|
||||
this.releaseDictation();
|
||||
this.scheduleIdleCloseIfReady();
|
||||
}
|
||||
|
||||
private handleMessage(message: DictationServerMessage): void {
|
||||
const dictationId = message.dictationId;
|
||||
if (!dictationId) {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (message.type) {
|
||||
case 'ack': {
|
||||
const pendingStart = this.pendingStarts.get(dictationId);
|
||||
if (pendingStart) {
|
||||
this.pendingStarts.delete(dictationId);
|
||||
pendingStart.resolve();
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'partial': {
|
||||
for (const listener of this.partialListeners) {
|
||||
listener(dictationId, message.text ?? '');
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'finish_accepted': {
|
||||
const pendingFinish = this.pendingFinishes.get(dictationId);
|
||||
if (pendingFinish && typeof message.timeoutMs === 'number') {
|
||||
if (pendingFinish.timeout) {
|
||||
clearTimeout(pendingFinish.timeout);
|
||||
}
|
||||
pendingFinish.timeout = setTimeout(() => {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
pendingFinish.reject(new Error('Timed out waiting for transcription'));
|
||||
}, message.timeoutMs + 5000);
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'final': {
|
||||
const pendingFinish = this.pendingFinishes.get(dictationId);
|
||||
if (pendingFinish) {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
pendingFinish.resolve({ text: message.text ?? '' });
|
||||
}
|
||||
this.scheduleIdleCloseIfReady();
|
||||
return;
|
||||
}
|
||||
case 'error': {
|
||||
const error = createStreamError(
|
||||
message.error || 'Dictation failed',
|
||||
message.retryable !== false,
|
||||
message.reasonCode,
|
||||
);
|
||||
const pendingStart = this.pendingStarts.get(dictationId);
|
||||
if (pendingStart) {
|
||||
this.pendingStarts.delete(dictationId);
|
||||
pendingStart.reject(error);
|
||||
}
|
||||
const pendingFinish = this.pendingFinishes.get(dictationId);
|
||||
if (pendingFinish) {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
pendingFinish.reject(error);
|
||||
}
|
||||
this.scheduleIdleCloseIfReady();
|
||||
return;
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
private send(message: object): boolean {
|
||||
if (!this.isConnected || !this.socket) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
this.socket.send(JSON.stringify(message));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private notifyConnection(connected: boolean): void {
|
||||
for (const listener of this.connectionListeners) {
|
||||
listener(connected);
|
||||
}
|
||||
}
|
||||
|
||||
private rejectAllPending(error: Error): void {
|
||||
for (const [dictationId, pending] of this.pendingStarts) {
|
||||
this.pendingStarts.delete(dictationId);
|
||||
pending.reject(error);
|
||||
}
|
||||
for (const [dictationId, pending] of this.pendingFinishes) {
|
||||
this.pendingFinishes.delete(dictationId);
|
||||
pending.reject(error);
|
||||
}
|
||||
this.activeDictations = 0;
|
||||
}
|
||||
|
||||
private releaseDictation(): void {
|
||||
this.activeDictations = Math.max(0, this.activeDictations - 1);
|
||||
this.scheduleIdleCloseIfReady();
|
||||
}
|
||||
|
||||
private scheduleIdleCloseIfReady(): void {
|
||||
if (this.activeDictations > 0 || this.pendingFinishes.size > 0 || this.pendingStarts.size > 0) {
|
||||
return;
|
||||
}
|
||||
this.clearIdleCloseTimer();
|
||||
this.idleCloseTimer = setTimeout(() => {
|
||||
if (this.activeDictations === 0 && this.pendingFinishes.size === 0 && this.pendingStarts.size === 0) {
|
||||
const socket = this.socket;
|
||||
this.socket = null;
|
||||
if (socket) {
|
||||
try {
|
||||
socket.close(1000, 'idle');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.notifyConnection(false);
|
||||
}
|
||||
}
|
||||
}, IDLE_CLOSE_DELAY_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runtime switch: close the socket and fail all in-flight dictations so
|
||||
* nothing keeps streaming to the previous runtime.
|
||||
*/
|
||||
cancelAllForRuntimeSwitch(): void {
|
||||
this.clearIdleCloseTimer();
|
||||
const socket = this.socket;
|
||||
this.socket = null;
|
||||
this.connectPromise = null;
|
||||
this.rejectAllPending(new Error('Runtime changed'));
|
||||
if (socket) {
|
||||
try {
|
||||
socket.close(1000, 'runtime switch');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.notifyConnection(false);
|
||||
}
|
||||
}
|
||||
|
||||
private clearIdleCloseTimer(): void {
|
||||
if (this.idleCloseTimer) {
|
||||
clearTimeout(this.idleCloseTimer);
|
||||
this.idleCloseTimer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const dictationClient = new DictationClient();
|
||||
|
||||
if (typeof window !== 'undefined') {
|
||||
window.addEventListener('openchamber:runtime-endpoint-changed', () => {
|
||||
// Drop the socket so the next dictation reconnects to the new runtime.
|
||||
dictationClient.cancelAllForRuntimeSwitch();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* Small, non-React state machine for dictation streaming.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Maintain an ordered buffer of base64 PCM segments
|
||||
* - Start/restart a dictation stream (dictationId)
|
||||
* - Send missing segments (seq) when connected
|
||||
* - Finish/cancel the stream
|
||||
*
|
||||
* Segments are retained until the dictation completes, which enables replay
|
||||
* after a connection drop (`resetStreamForReplay()` + `finish()`).
|
||||
*/
|
||||
|
||||
import type { DictationClient, DictationStartOptions } from './dictation-client';
|
||||
|
||||
const MAX_CHUNKS_PER_FLUSH_TURN = 128;
|
||||
|
||||
const PCM_DICTATION_FORMAT = 'audio/pcm;rate=16000;bits=16';
|
||||
|
||||
const waitForNextFlushTurn = (): Promise<void> => new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
const createDictationIdDefault = (): string => {
|
||||
const rand = Math.random().toString(36).slice(2, 10);
|
||||
return `dic_${Date.now().toString(16)}${rand}`;
|
||||
};
|
||||
|
||||
export interface DictationFinishResult {
|
||||
dictationId: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export class DictationStreamSender {
|
||||
private readonly client: DictationClient;
|
||||
private readonly format: string;
|
||||
private readonly createDictationId: () => string;
|
||||
private getStartOptions: () => DictationStartOptions;
|
||||
|
||||
private dictationId: string | null = null;
|
||||
private sendSeq = 0;
|
||||
private segments: string[] = [];
|
||||
private streamReady = false;
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private drainWaiters: Array<() => void> = [];
|
||||
|
||||
private startGeneration = 0;
|
||||
private startPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(params: {
|
||||
client: DictationClient;
|
||||
getStartOptions: () => DictationStartOptions;
|
||||
format?: string;
|
||||
createDictationId?: () => string;
|
||||
}) {
|
||||
this.client = params.client;
|
||||
this.format = params.format ?? PCM_DICTATION_FORMAT;
|
||||
this.getStartOptions = params.getStartOptions;
|
||||
this.createDictationId = params.createDictationId ?? createDictationIdDefault;
|
||||
}
|
||||
|
||||
getDictationId(): string | null {
|
||||
return this.dictationId;
|
||||
}
|
||||
|
||||
getFinalSeq(): number {
|
||||
return this.segments.length - 1;
|
||||
}
|
||||
|
||||
hasSegments(): boolean {
|
||||
return this.segments.length > 0;
|
||||
}
|
||||
|
||||
clearAll(): void {
|
||||
this.clearScheduledFlush();
|
||||
this.dictationId = null;
|
||||
this.sendSeq = 0;
|
||||
this.segments = [];
|
||||
this.streamReady = false;
|
||||
this.startPromise = null;
|
||||
this.startGeneration += 1;
|
||||
}
|
||||
|
||||
resetStreamForReplay(): void {
|
||||
this.clearScheduledFlush();
|
||||
this.dictationId = null;
|
||||
this.sendSeq = 0;
|
||||
this.streamReady = false;
|
||||
this.startPromise = null;
|
||||
this.startGeneration += 1;
|
||||
}
|
||||
|
||||
enqueueSegment(base64Pcm: string): void {
|
||||
this.segments.push(base64Pcm);
|
||||
|
||||
if (!this.client.isConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.dictationId) {
|
||||
if (!this.startPromise) {
|
||||
void this.restartStream().catch(() => {
|
||||
// Start failures surface through finish(); segments are retained.
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this.flush();
|
||||
}
|
||||
|
||||
flush(): number {
|
||||
const dictationId = this.dictationId;
|
||||
if (!this.client.isConnected || !dictationId || !this.streamReady) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let sent = 0;
|
||||
while (this.sendSeq < this.segments.length && sent < MAX_CHUNKS_PER_FLUSH_TURN) {
|
||||
const seq = this.sendSeq;
|
||||
const audio = this.segments[seq];
|
||||
if (!this.client.sendDictationStreamChunk(dictationId, seq, audio)) {
|
||||
break;
|
||||
}
|
||||
this.sendSeq = seq + 1;
|
||||
sent += 1;
|
||||
}
|
||||
if (this.hasPendingSegments()) {
|
||||
this.scheduleFlush();
|
||||
} else {
|
||||
this.resolveDrainWaiters();
|
||||
}
|
||||
return sent;
|
||||
}
|
||||
|
||||
async restartStream(): Promise<void> {
|
||||
this.startGeneration += 1;
|
||||
const generation = this.startGeneration;
|
||||
|
||||
const dictationId = this.createDictationId();
|
||||
this.dictationId = dictationId;
|
||||
this.sendSeq = 0;
|
||||
this.streamReady = false;
|
||||
|
||||
const start = (async () => {
|
||||
await this.client.startDictationStream(dictationId, this.format, this.getStartOptions());
|
||||
if (this.startGeneration !== generation) {
|
||||
return;
|
||||
}
|
||||
if (this.dictationId !== dictationId) {
|
||||
return;
|
||||
}
|
||||
this.streamReady = true;
|
||||
this.flush();
|
||||
})()
|
||||
.catch((error) => {
|
||||
// Keep segments for retry, but clear the stream so finish can error cleanly.
|
||||
if (this.startGeneration === generation && this.dictationId === dictationId) {
|
||||
this.dictationId = null;
|
||||
this.streamReady = false;
|
||||
}
|
||||
throw error;
|
||||
})
|
||||
.finally(() => {
|
||||
if (this.startPromise === start) {
|
||||
this.startPromise = null;
|
||||
}
|
||||
});
|
||||
|
||||
this.startPromise = start;
|
||||
await start;
|
||||
}
|
||||
|
||||
async finish(finalSeq: number): Promise<DictationFinishResult> {
|
||||
if (!this.dictationId) {
|
||||
await this.restartStream();
|
||||
}
|
||||
if (this.startPromise) {
|
||||
await this.startPromise;
|
||||
}
|
||||
|
||||
const dictationId = this.dictationId;
|
||||
if (!dictationId || !this.streamReady) {
|
||||
throw new Error('Failed to start dictation stream');
|
||||
}
|
||||
|
||||
this.flush();
|
||||
await this.waitForFlushDrain();
|
||||
const result = await this.client.finishDictationStream(dictationId, finalSeq);
|
||||
return { dictationId, text: result.text };
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
const dictationId = this.dictationId;
|
||||
if (this.client.isConnected && dictationId) {
|
||||
this.client.cancelDictationStream(dictationId);
|
||||
}
|
||||
this.resetStreamForReplay();
|
||||
}
|
||||
|
||||
private hasPendingSegments(): boolean {
|
||||
return this.sendSeq < this.segments.length;
|
||||
}
|
||||
|
||||
private scheduleFlush(): void {
|
||||
if (this.flushTimer) {
|
||||
return;
|
||||
}
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flush();
|
||||
}, 0);
|
||||
}
|
||||
|
||||
private clearScheduledFlush(): void {
|
||||
if (!this.flushTimer) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
|
||||
private async waitForFlushDrain(): Promise<void> {
|
||||
while (this.hasPendingSegments()) {
|
||||
if (!this.client.isConnected || !this.dictationId || !this.streamReady) {
|
||||
throw new Error('Failed to flush dictation stream');
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
this.drainWaiters.push(resolve);
|
||||
});
|
||||
await waitForNextFlushTurn();
|
||||
}
|
||||
}
|
||||
|
||||
private resolveDrainWaiters(): void {
|
||||
const waiters = this.drainWaiters;
|
||||
this.drainWaiters = [];
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
/**
|
||||
* Microphone capture for dictation.
|
||||
*
|
||||
* Captures mono audio via getUserMedia, taps it with a ScriptProcessorNode
|
||||
* (universally supported, including iOS WKWebView), resamples Float32 to
|
||||
* 16 kHz PCM16LE, and emits ~1-second base64 chunks plus a normalized RMS
|
||||
* volume for the level meter.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
|
||||
export interface DictationAudioSourceConfig {
|
||||
onPcmSegment: (base64Pcm: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
export interface DictationAudioSource {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
const OUTPUT_RATE = 16000;
|
||||
const CHUNK_SAMPLES = OUTPUT_RATE; // ~1s per chunk
|
||||
|
||||
const getAudioContextCtor = (): typeof AudioContext | null => {
|
||||
if (typeof window === 'undefined') {
|
||||
return null;
|
||||
}
|
||||
const win = window as typeof window & { webkitAudioContext?: typeof AudioContext };
|
||||
return win.AudioContext || win.webkitAudioContext || null;
|
||||
};
|
||||
|
||||
const floatToInt16 = (sample: number): number => {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
};
|
||||
|
||||
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
|
||||
if (input.length === 0) {
|
||||
return new Int16Array(0);
|
||||
}
|
||||
if (inputRate === outputRate) {
|
||||
const out = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
out[i] = floatToInt16(input[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i++) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
out[i] = floatToInt16(input[i0] * (1 - frac) + input[i1] * frac);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => {
|
||||
if (a.length === 0) {
|
||||
return b;
|
||||
}
|
||||
if (b.length === 0) {
|
||||
return a;
|
||||
}
|
||||
const out = new Int16Array(a.length + b.length);
|
||||
out.set(a, 0);
|
||||
out.set(b, a.length);
|
||||
return out;
|
||||
};
|
||||
|
||||
const int16ToBase64 = (pcm: Int16Array): string => {
|
||||
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||
let binary = '';
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
};
|
||||
|
||||
interface CaptureGraph {
|
||||
stream: MediaStream | null;
|
||||
context: AudioContext | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
pending: Int16Array;
|
||||
started: boolean;
|
||||
}
|
||||
|
||||
const emptyGraph = (): CaptureGraph => ({
|
||||
stream: null,
|
||||
context: null,
|
||||
source: null,
|
||||
processor: null,
|
||||
gain: null,
|
||||
pending: new Int16Array(0),
|
||||
started: false,
|
||||
});
|
||||
|
||||
const safeDisconnect = (node: AudioNode | null): void => {
|
||||
if (!node) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
node.disconnect();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
};
|
||||
|
||||
export const isDictationCaptureSupported = (): boolean => {
|
||||
if (typeof navigator === 'undefined' || typeof window === 'undefined') {
|
||||
return false;
|
||||
}
|
||||
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
||||
return false;
|
||||
}
|
||||
return getAudioContextCtor() !== null;
|
||||
};
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const [volume, setVolume] = useState(0);
|
||||
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
useEffect(() => {
|
||||
onPcmSegmentRef.current = config.onPcmSegment;
|
||||
onErrorRef.current = config.onError;
|
||||
}, [config.onPcmSegment, config.onError]);
|
||||
|
||||
const graphRef = useRef<CaptureGraph>(emptyGraph());
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (graphRef.current.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
typeof navigator === 'undefined' ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== 'function'
|
||||
) {
|
||||
throw new Error('Microphone capture is not supported in this environment');
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error('AudioContext unavailable');
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
try {
|
||||
if (context.state === 'suspended') {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
graphRef.current = {
|
||||
stream,
|
||||
context,
|
||||
source,
|
||||
processor,
|
||||
gain,
|
||||
pending: new Int16Array(0),
|
||||
started: true,
|
||||
};
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
const graph = graphRef.current;
|
||||
if (!graph.started) {
|
||||
return;
|
||||
}
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
sumSquares += input[i] * input[i];
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
setVolume(Math.min(1, Math.max(0, rms * 2)));
|
||||
|
||||
const next = resampleToPcm16(input, context.sampleRate, OUTPUT_RATE);
|
||||
graph.pending = concatInt16(graph.pending, next);
|
||||
|
||||
while (graph.pending.length >= CHUNK_SAMPLES) {
|
||||
const chunk = graph.pending.slice(0, CHUNK_SAMPLES);
|
||||
graph.pending = graph.pending.slice(CHUNK_SAMPLES);
|
||||
onPcmSegmentRef.current(int16ToBase64(chunk));
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
} catch (error) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
});
|
||||
try {
|
||||
await context.close();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
graphRef.current = emptyGraph();
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
const graph = graphRef.current;
|
||||
graph.started = false;
|
||||
setVolume(0);
|
||||
|
||||
if (graph.processor) {
|
||||
try {
|
||||
graph.processor.onaudioprocess = null;
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
safeDisconnect(graph.processor);
|
||||
safeDisconnect(graph.source);
|
||||
safeDisconnect(graph.gain);
|
||||
if (graph.stream) {
|
||||
graph.stream.getTracks().forEach((track) => {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
});
|
||||
}
|
||||
const pending = graph.pending;
|
||||
graph.pending = new Int16Array(0);
|
||||
if (pending.length > 0) {
|
||||
onPcmSegmentRef.current(int16ToBase64(pending));
|
||||
}
|
||||
|
||||
if (graph.context) {
|
||||
try {
|
||||
await graph.context.close();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
// A new capture may have started while the old context was closing;
|
||||
// only clear the ref if it still points at the graph we tore down.
|
||||
if (graphRef.current === graph) {
|
||||
graphRef.current = emptyGraph();
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
void stop().catch((err) => {
|
||||
onErrorRef.current?.(err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
start: async () => {
|
||||
try {
|
||||
await start();
|
||||
} catch (err) {
|
||||
const normalized = err instanceof Error ? err : new Error(String(err));
|
||||
onErrorRef.current?.(normalized);
|
||||
throw normalized;
|
||||
}
|
||||
},
|
||||
stop,
|
||||
volume,
|
||||
}),
|
||||
[start, stop, volume],
|
||||
);
|
||||
}
|
||||
@@ -1518,16 +1518,33 @@ export const settingsDict = {
|
||||
'settings.notifications.page.toast.backgroundDisabled': 'Background notifications disabled',
|
||||
'settings.notifications.page.testNotification.title': 'Test Notification',
|
||||
'settings.notifications.page.testNotification.body': 'This is a test notification from OpenChamber.',
|
||||
'settings.voice.page.section.voiceSetup': 'Voice Setup',
|
||||
'settings.voice.page.section.speechRecognition': 'Speech Recognition',
|
||||
'settings.voice.page.field.enableVoiceInput': 'Enable voice input',
|
||||
'settings.voice.page.field.enableVoiceInputAria': 'Enable voice input (dictation)',
|
||||
'settings.voice.page.section.playbackAndSummary': 'Playback',
|
||||
'settings.voice.page.field.enableVoiceModeAria': 'Enable voice mode',
|
||||
'settings.voice.page.field.enableVoiceMode': 'Enable Voice Mode',
|
||||
'settings.voice.page.field.provider': 'Provider',
|
||||
'settings.voice.page.provider.browser': 'Browser',
|
||||
'settings.voice.page.provider.custom': 'Custom',
|
||||
'settings.voice.page.provider.say': 'Say',
|
||||
'settings.voice.page.provider.server': 'Server',
|
||||
'settings.voice.page.provider.local': 'Local',
|
||||
'settings.voice.page.tooltip.sttLocal': 'On-device transcription on the OpenChamber server. Models download automatically; no API key needed.',
|
||||
'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro, English). The model downloads automatically; no API key needed.',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (English)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 European languages)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingual)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (multilingual)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': 'Best for English',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': 'Best for multilingual',
|
||||
'settings.voice.page.stt.meta.accuracy': 'Accuracy',
|
||||
'settings.voice.page.stt.meta.speed': 'Speed',
|
||||
'settings.voice.page.stt.modelInstalled': 'Model installed',
|
||||
'settings.voice.page.stt.modelDownloading': 'Downloading model...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': 'Downloading model... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': 'Model not downloaded',
|
||||
'settings.voice.page.stt.modelDownload': 'Download',
|
||||
'settings.voice.page.stt.modelDelete': 'Delete model',
|
||||
'settings.voice.page.stt.modelRetry': 'Retry',
|
||||
'settings.voice.page.provider.wasm': 'Local',
|
||||
'settings.voice.page.stt.wasmModel': 'Whisper Model',
|
||||
'settings.voice.page.stt.wasmLoaded': 'Model loaded, ready',
|
||||
|
||||
@@ -1873,6 +1873,18 @@ export const dict = {
|
||||
'chat.chatInput.placeholder.chat': '@ for files/agents; / for commands and skills; ! for shell; # for snippets',
|
||||
'chat.chatInput.placeholder.chatCompact': 'Use @ / ! # for helpers',
|
||||
'chat.chatInput.placeholder.selectSession': 'Select or create a session to start chatting',
|
||||
'chat.dictation.start': 'Start dictation',
|
||||
'chat.dictation.overlayAria': 'Dictation',
|
||||
'chat.dictation.downloadingModel': 'Downloading speech model...',
|
||||
'chat.dictation.downloadingModelProgress': 'Downloading speech model... {percent}%',
|
||||
'chat.dictation.listening': 'Listening...',
|
||||
'chat.dictation.processing': 'Transcribing...',
|
||||
'chat.dictation.failed': 'Transcription failed',
|
||||
'chat.dictation.cancel': 'Discard dictation',
|
||||
'chat.dictation.insert': 'Insert transcript',
|
||||
'chat.dictation.insertAndSend': 'Insert and send',
|
||||
'chat.dictation.retry': 'Retry transcription',
|
||||
'chat.dictation.discard': 'Discard recording',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ Add new snippet',
|
||||
'chat.snippetAutocomplete.empty': 'No snippets found',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ navigate • Enter select • Esc close',
|
||||
|
||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
||||
"settings.notifications.page.toast.backgroundDisabled": "Notificaciones de fondo deshabilitadas",
|
||||
"settings.notifications.page.testNotification.title": "Notificación de prueba",
|
||||
"settings.notifications.page.testNotification.body": "Esta es una notificación de prueba de OpenChamber.",
|
||||
"settings.voice.page.section.voiceSetup": "Configuración de voz",
|
||||
"settings.voice.page.section.speechRecognition": "Reconocimiento de voz",
|
||||
"settings.voice.page.field.enableVoiceInput": "Habilitar entrada de voz",
|
||||
"settings.voice.page.field.enableVoiceInputAria": "Habilitar entrada de voz (dictado)",
|
||||
"settings.voice.page.section.playbackAndSummary": "Reproducción",
|
||||
"settings.voice.page.field.enableVoiceModeAria": "Habilitar modo de voz",
|
||||
"settings.voice.page.field.enableVoiceMode": "Habilitar modo de voz",
|
||||
"settings.voice.page.field.provider": "Proveedor",
|
||||
"settings.voice.page.provider.browser": "Navegador",
|
||||
"settings.voice.page.provider.custom": "Personalizado",
|
||||
"settings.voice.page.provider.say": "Decir",
|
||||
"settings.voice.page.provider.server": "Servidor",
|
||||
"settings.voice.page.provider.local": "Local",
|
||||
"settings.voice.page.tooltip.sttLocal": "Transcripción local en el servidor de OpenChamber. Los modelos se descargan automáticamente; no se necesita clave de API.",
|
||||
"settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro, inglés). El modelo se descarga automáticamente; no se necesita clave de API.",
|
||||
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglés)",
|
||||
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeos)",
|
||||
"settings.voice.page.stt.model.whisperBase": "Whisper base (multilingüe)",
|
||||
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (multilingüe)",
|
||||
"settings.voice.page.stt.badge.bestForEnglish": "Mejor para inglés",
|
||||
"settings.voice.page.stt.badge.bestForMultilingual": "Mejor multilingüe",
|
||||
"settings.voice.page.stt.meta.accuracy": "Precisión",
|
||||
"settings.voice.page.stt.meta.speed": "Velocidad",
|
||||
"settings.voice.page.stt.modelInstalled": "Modelo instalado",
|
||||
"settings.voice.page.stt.modelDownloading": "Descargando modelo...",
|
||||
"settings.voice.page.stt.modelDownloadingProgress": "Descargando modelo... {percent}%",
|
||||
"settings.voice.page.stt.modelNotInstalled": "Modelo no descargado",
|
||||
"settings.voice.page.stt.modelDownload": "Descargar",
|
||||
"settings.voice.page.stt.modelDelete": "Eliminar modelo",
|
||||
"settings.voice.page.stt.modelRetry": "Reintentar",
|
||||
"settings.voice.page.provider.wasm": "Local",
|
||||
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
|
||||
"settings.voice.page.stt.wasmLoaded": "Modelo cargado",
|
||||
|
||||
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'diffView.reviewDialog.actions.starting': 'Iniciando...',
|
||||
'diffView.reviewDialog.toast.noSessionDirectory': 'El directorio de la sesión no está disponible',
|
||||
'diffView.reviewDialog.toast.startFailed': 'No se pudo iniciar el flujo de revisión',
|
||||
'chat.dictation.start': 'Iniciar dictado',
|
||||
'chat.dictation.overlayAria': 'Dictado',
|
||||
'chat.dictation.downloadingModel': 'Descargando el modelo de voz...',
|
||||
'chat.dictation.downloadingModelProgress': 'Descargando el modelo de voz... {percent}%',
|
||||
'chat.dictation.listening': 'Escuchando...',
|
||||
'chat.dictation.processing': 'Transcribiendo...',
|
||||
'chat.dictation.failed': 'La transcripción falló',
|
||||
'chat.dictation.cancel': 'Descartar dictado',
|
||||
'chat.dictation.insert': 'Insertar transcripción',
|
||||
'chat.dictation.insertAndSend': 'Insertar y enviar',
|
||||
'chat.dictation.retry': 'Reintentar transcripción',
|
||||
'chat.dictation.discard': 'Descartar grabación',
|
||||
'chat.history.loadOlder': 'Cargar mensajes anteriores',
|
||||
'chat.autoReview.title': 'El ciclo de revisión de código está en curso',
|
||||
'chat.autoReview.status.waitingForReviewer': 'Esperando al revisor',
|
||||
|
||||
@@ -1468,16 +1468,33 @@ export const settingsDict = {
|
||||
'settings.notifications.page.toast.backgroundDisabled': 'Notifications en arrière-plan désactivées',
|
||||
'settings.notifications.page.testNotification.title': 'Avis de test',
|
||||
'settings.notifications.page.testNotification.body': 'Il s\'agit d\'une notification de test de OpenChamber.',
|
||||
'settings.voice.page.section.voiceSetup': 'Configuration vocale',
|
||||
'settings.voice.page.section.speechRecognition': 'Reconnaissance vocale',
|
||||
'settings.voice.page.field.enableVoiceInput': 'Activer la saisie vocale',
|
||||
'settings.voice.page.field.enableVoiceInputAria': 'Activer la saisie vocale (dictée)',
|
||||
'settings.voice.page.section.playbackAndSummary': 'Lecture',
|
||||
'settings.voice.page.field.enableVoiceModeAria': 'Activer le mode vocal',
|
||||
'settings.voice.page.field.enableVoiceMode': 'Activer le mode vocal',
|
||||
'settings.voice.page.field.provider': 'Fournisseur',
|
||||
'settings.voice.page.provider.browser': 'Navigateur',
|
||||
'settings.voice.page.provider.custom': 'Personnalisé',
|
||||
'settings.voice.page.provider.say': 'Dire',
|
||||
'settings.voice.page.provider.server': 'Serveur',
|
||||
'settings.voice.page.provider.local': 'Local',
|
||||
'settings.voice.page.tooltip.sttLocal': 'Transcription locale sur le serveur OpenChamber. Les modèles se téléchargent automatiquement ; aucune clé d\'API requise.',
|
||||
'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro, anglais). Le modèle se télécharge automatiquement ; aucune clé d’API requise.',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (anglais)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 langues européennes)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingue)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (multilingue)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': 'Idéal pour l’anglais',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': 'Idéal multilingue',
|
||||
'settings.voice.page.stt.meta.accuracy': 'Précision',
|
||||
'settings.voice.page.stt.meta.speed': 'Vitesse',
|
||||
'settings.voice.page.stt.modelInstalled': 'Modèle installé',
|
||||
'settings.voice.page.stt.modelDownloading': 'Téléchargement du modèle...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': 'Téléchargement du modèle... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': 'Modèle non téléchargé',
|
||||
'settings.voice.page.stt.modelDownload': 'Télécharger',
|
||||
'settings.voice.page.stt.modelDelete': 'Supprimer le modèle',
|
||||
'settings.voice.page.stt.modelRetry': 'Réessayer',
|
||||
'settings.voice.page.provider.wasm': 'Local',
|
||||
'settings.voice.page.stt.wasmModel': 'Modèle Whisper',
|
||||
'settings.voice.page.stt.wasmLoaded': 'Modèle chargé, prêt',
|
||||
|
||||
@@ -1669,6 +1669,18 @@ export const dict = {
|
||||
'chat.chatInput.placeholder.chat': '@ pour les fichiers/agents ; / pour les commandes et les skills ; ! pour shell ; # pour les extraits',
|
||||
'chat.chatInput.placeholder.chatCompact': 'Utiliser @ / ! # pour les aides',
|
||||
'chat.chatInput.placeholder.selectSession': 'Sélectionnez ou créez une session pour commencer à discuter',
|
||||
'chat.dictation.start': 'Démarrer la dictée',
|
||||
'chat.dictation.overlayAria': 'Dictée',
|
||||
'chat.dictation.downloadingModel': 'Téléchargement du modèle vocal...',
|
||||
'chat.dictation.downloadingModelProgress': 'Téléchargement du modèle vocal... {percent}%',
|
||||
'chat.dictation.listening': 'Écoute...',
|
||||
'chat.dictation.processing': 'Transcription...',
|
||||
'chat.dictation.failed': 'Échec de la transcription',
|
||||
'chat.dictation.cancel': 'Abandonner la dictée',
|
||||
'chat.dictation.insert': 'Insérer la transcription',
|
||||
'chat.dictation.insertAndSend': 'Insérer et envoyer',
|
||||
'chat.dictation.retry': 'Réessayer la transcription',
|
||||
'chat.dictation.discard': 'Abandonner l\'enregistrement',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ Ajouter un nouvel extrait',
|
||||
'chat.snippetAutocomplete.empty': 'Aucun extrait trouvé',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ naviguer • Entrer sélectionner • Esc fermer',
|
||||
|
||||
@@ -1518,16 +1518,33 @@ export const settingsDict = {
|
||||
'settings.notifications.page.toast.backgroundDisabled': 'バックグラウンド通知を無効化しました',
|
||||
'settings.notifications.page.testNotification.title': 'テスト通知',
|
||||
'settings.notifications.page.testNotification.body': 'これは OpenChamber からのテスト通知です。',
|
||||
'settings.voice.page.section.voiceSetup': '音声設定',
|
||||
'settings.voice.page.section.speechRecognition': '音声認識',
|
||||
'settings.voice.page.field.enableVoiceInput': '音声入力を有効にする',
|
||||
'settings.voice.page.field.enableVoiceInputAria': '音声入力(ディクテーション)を有効にする',
|
||||
'settings.voice.page.section.playbackAndSummary': '再生',
|
||||
'settings.voice.page.field.enableVoiceModeAria': '音声モードを有効化',
|
||||
'settings.voice.page.field.enableVoiceMode': '音声モードを有効化',
|
||||
'settings.voice.page.field.provider': 'プロバイダー',
|
||||
'settings.voice.page.provider.browser': 'ブラウザ',
|
||||
'settings.voice.page.provider.custom': 'カスタム',
|
||||
'settings.voice.page.provider.say': 'Say',
|
||||
'settings.voice.page.provider.server': 'サーバー',
|
||||
'settings.voice.page.provider.local': 'ローカル',
|
||||
'settings.voice.page.tooltip.sttLocal': 'OpenChamber サーバー上でローカルに文字起こしします。モデルは自動でダウンロードされ、API キーは不要です。',
|
||||
'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します(Kokoro、英語)。モデルは自動でダウンロードされ、API キーは不要です。',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英語)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(ヨーロッパ25言語)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多言語)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多言語)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': '英語に最適',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': '多言語に最適',
|
||||
'settings.voice.page.stt.meta.accuracy': '精度',
|
||||
'settings.voice.page.stt.meta.speed': '速度',
|
||||
'settings.voice.page.stt.modelInstalled': 'モデルはインストール済み',
|
||||
'settings.voice.page.stt.modelDownloading': 'モデルをダウンロード中...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': 'モデルをダウンロード中... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': 'モデル未ダウンロード',
|
||||
'settings.voice.page.stt.modelDownload': 'ダウンロード',
|
||||
'settings.voice.page.stt.modelDelete': 'モデルを削除',
|
||||
'settings.voice.page.stt.modelRetry': '再試行',
|
||||
'settings.voice.page.provider.wasm': 'ローカル',
|
||||
'settings.voice.page.stt.wasmModel': 'Whisper モデル',
|
||||
'settings.voice.page.stt.wasmLoaded': 'モデル読み込み完了',
|
||||
|
||||
@@ -1869,6 +1869,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'chat.chatInput.placeholder.chat': '@でファイル/エージェント、/でコマンド/スキル、!でシェル、#でスニペット',
|
||||
'chat.chatInput.placeholder.chatCompact': '@ / ! # でヘルパーを使用',
|
||||
'chat.chatInput.placeholder.selectSession': 'セッションを選択または作成してチャットを開始',
|
||||
'chat.dictation.start': '音声入力を開始',
|
||||
'chat.dictation.overlayAria': '音声入力',
|
||||
'chat.dictation.downloadingModel': '音声モデルをダウンロード中...',
|
||||
'chat.dictation.downloadingModelProgress': '音声モデルをダウンロード中... {percent}%',
|
||||
'chat.dictation.listening': '聞き取り中...',
|
||||
'chat.dictation.processing': '文字起こし中...',
|
||||
'chat.dictation.failed': '文字起こしに失敗しました',
|
||||
'chat.dictation.cancel': '音声入力を破棄',
|
||||
'chat.dictation.insert': 'テキストを挿入',
|
||||
'chat.dictation.insertAndSend': '挿入して送信',
|
||||
'chat.dictation.retry': '文字起こしを再試行',
|
||||
'chat.dictation.discard': '録音を破棄',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ 新しいスニペットを追加',
|
||||
'chat.snippetAutocomplete.empty': 'スニペットが見つかりません',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ 移動 • Enter 選択 • Esc 閉じる',
|
||||
|
||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
||||
'settings.notifications.page.toast.backgroundDisabled': '백그라운드 알림이 비활성화되었습니다',
|
||||
'settings.notifications.page.testNotification.title': '테스트 알림',
|
||||
'settings.notifications.page.testNotification.body': 'OpenChamber의 테스트 알림입니다.',
|
||||
'settings.voice.page.section.voiceSetup': '음성',
|
||||
'settings.voice.page.section.speechRecognition': '음성 인식',
|
||||
'settings.voice.page.field.enableVoiceInput': '음성 입력 사용',
|
||||
'settings.voice.page.field.enableVoiceInputAria': '음성 입력(받아쓰기) 사용',
|
||||
'settings.voice.page.section.playbackAndSummary': '재생',
|
||||
'settings.voice.page.field.enableVoiceModeAria': '음성 모드 활성화',
|
||||
'settings.voice.page.field.enableVoiceMode': '음성 모드 활성화',
|
||||
'settings.voice.page.field.provider': '프로바이더',
|
||||
'settings.voice.page.provider.browser': '브라우저',
|
||||
'settings.voice.page.provider.custom': '사용자 정의',
|
||||
'settings.voice.page.provider.say': 'Say',
|
||||
'settings.voice.page.provider.server': '서버',
|
||||
'settings.voice.page.provider.local': '로컬',
|
||||
'settings.voice.page.tooltip.sttLocal': 'OpenChamber 서버에서 로컬로 변환합니다. 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.',
|
||||
'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(Kokoro, 영어). 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (영어)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (유럽 25개 언어)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base (다국어)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (다국어)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': '영어에 최적',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': '다국어에 최적',
|
||||
'settings.voice.page.stt.meta.accuracy': '정확도',
|
||||
'settings.voice.page.stt.meta.speed': '속도',
|
||||
'settings.voice.page.stt.modelInstalled': '모델 설치됨',
|
||||
'settings.voice.page.stt.modelDownloading': '모델 다운로드 중...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': '모델 다운로드 중... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': '모델이 다운로드되지 않음',
|
||||
'settings.voice.page.stt.modelDownload': '다운로드',
|
||||
'settings.voice.page.stt.modelDelete': '모델 삭제',
|
||||
'settings.voice.page.stt.modelRetry': '다시 시도',
|
||||
'settings.voice.page.provider.wasm': '로컬',
|
||||
'settings.voice.page.stt.wasmModel': 'Whisper 모델',
|
||||
'settings.voice.page.stt.wasmLoaded': '모델 로드됨',
|
||||
|
||||
@@ -1873,6 +1873,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'chat.chatInput.placeholder.chat': '@ 파일/에이전트; / 명령 및 스킬; ! shell; # 스니펫',
|
||||
'chat.chatInput.placeholder.chatCompact': '@ / ! # 도우미 사용',
|
||||
'chat.chatInput.placeholder.selectSession': '채팅을 시작할 세션을 선택하거나 새로 만드세요',
|
||||
'chat.dictation.start': '받아쓰기 시작',
|
||||
'chat.dictation.overlayAria': '받아쓰기',
|
||||
'chat.dictation.downloadingModel': '음성 모델 다운로드 중...',
|
||||
'chat.dictation.downloadingModelProgress': '음성 모델 다운로드 중... {percent}%',
|
||||
'chat.dictation.listening': '듣는 중...',
|
||||
'chat.dictation.processing': '변환 중...',
|
||||
'chat.dictation.failed': '변환에 실패했습니다',
|
||||
'chat.dictation.cancel': '받아쓰기 취소',
|
||||
'chat.dictation.insert': '텍스트 삽입',
|
||||
'chat.dictation.insertAndSend': '삽입 후 전송',
|
||||
'chat.dictation.retry': '변환 다시 시도',
|
||||
'chat.dictation.discard': '녹음 삭제',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ 새 스니펫 추가',
|
||||
'chat.snippetAutocomplete.empty': '스니펫을 찾을 수 없음',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ 이동 • Enter 선택 • Esc 닫기',
|
||||
|
||||
@@ -1743,8 +1743,6 @@ export const settingsDict = {
|
||||
'settings.voice.page.field.apiKeyHintUsingConfig': 'Użyto klucza z konfiguracji',
|
||||
'settings.voice.page.field.auto': 'Auto',
|
||||
'settings.voice.page.field.configuredAbove': 'Skonfigurowano powyżej',
|
||||
'settings.voice.page.field.enableVoiceMode': 'Włącz tryb głosowy',
|
||||
'settings.voice.page.field.enableVoiceModeAria': 'Włącz tryb głosowy',
|
||||
'settings.voice.page.field.language': 'Język',
|
||||
'settings.voice.page.field.messageReadAloudButton': 'Przycisk czytania wiadomości na głos',
|
||||
'settings.voice.page.field.messageReadAloudButtonAria': 'Przycisk czytania wiadomości na głos',
|
||||
@@ -1782,6 +1780,24 @@ export const settingsDict = {
|
||||
'settings.voice.page.provider.custom': 'Własny',
|
||||
'settings.voice.page.provider.say': 'Say',
|
||||
'settings.voice.page.provider.server': 'Serwer',
|
||||
'settings.voice.page.provider.local': 'Lokalny',
|
||||
'settings.voice.page.tooltip.sttLocal': 'Transkrypcja lokalna na serwerze OpenChamber. Modele pobierają się automatycznie; klucz API nie jest potrzebny.',
|
||||
'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro, angielski). Model pobiera się automatycznie; klucz API nie jest potrzebny.',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (angielski)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 języków europejskich)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base (wielojęzyczny)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (wielojęzyczny)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': 'Najlepszy do angielskiego',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': 'Najlepszy wielojęzyczny',
|
||||
'settings.voice.page.stt.meta.accuracy': 'Dokładność',
|
||||
'settings.voice.page.stt.meta.speed': 'Szybkość',
|
||||
'settings.voice.page.stt.modelInstalled': 'Model zainstalowany',
|
||||
'settings.voice.page.stt.modelDownloading': 'Pobieranie modelu...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': 'Pobieranie modelu... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': 'Model niepobrany',
|
||||
'settings.voice.page.stt.modelDownload': 'Pobierz',
|
||||
'settings.voice.page.stt.modelDelete': 'Usuń model',
|
||||
'settings.voice.page.stt.modelRetry': 'Ponów',
|
||||
'settings.voice.page.provider.wasm': 'Lokalny',
|
||||
'settings.voice.page.stt.wasmModel': 'Model Whisper',
|
||||
'settings.voice.page.stt.wasmLoaded': 'Model załadowany',
|
||||
@@ -1792,7 +1808,8 @@ export const settingsDict = {
|
||||
'settings.voice.page.stt.wasmRetry': 'Spróbuj ponownie',
|
||||
'settings.voice.page.section.playbackAndSummary': 'Odtwarzanie',
|
||||
'settings.voice.page.section.speechRecognition': 'Rozpoznawanie mowy',
|
||||
'settings.voice.page.section.voiceSetup': 'Konfiguracja głosu',
|
||||
'settings.voice.page.field.enableVoiceInput': 'Włącz wprowadzanie głosowe',
|
||||
'settings.voice.page.field.enableVoiceInputAria': 'Włącz wprowadzanie głosowe (dyktowanie)',
|
||||
'settings.voice.page.tooltip.browser': 'Darmowe, offline, ograniczone wsparcie mobilne.',
|
||||
'settings.voice.page.tooltip.custom': 'Serwer zgodny z OpenAI (na przykład Kokoro).',
|
||||
'settings.voice.page.tooltip.openai': 'Wysoka jakość, gotowe na urządzenia mobilne, wymaga klucza API.',
|
||||
|
||||
@@ -1091,6 +1091,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'chat.chatInput.placeholder.chatCompact': 'Użyj @ / ! # dla pomocników',
|
||||
'chat.chatInput.placeholder.selectSession': 'Wybierz lub utwórz sesję, aby zacząć czatować',
|
||||
'chat.chatInput.placeholder.shell': 'Wpisz polecenie powłoki...',
|
||||
'chat.dictation.start': 'Rozpocznij dyktowanie',
|
||||
'chat.dictation.overlayAria': 'Dyktowanie',
|
||||
'chat.dictation.downloadingModel': 'Pobieranie modelu mowy...',
|
||||
'chat.dictation.downloadingModelProgress': 'Pobieranie modelu mowy... {percent}%',
|
||||
'chat.dictation.listening': 'Słucham...',
|
||||
'chat.dictation.processing': 'Transkrybowanie...',
|
||||
'chat.dictation.failed': 'Transkrypcja nie powiodła się',
|
||||
'chat.dictation.cancel': 'Odrzuć dyktowanie',
|
||||
'chat.dictation.insert': 'Wstaw transkrypcję',
|
||||
'chat.dictation.insertAndSend': 'Wstaw i wyślij',
|
||||
'chat.dictation.retry': 'Ponów transkrypcję',
|
||||
'chat.dictation.discard': 'Odrzuć nagranie',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ Dodaj nowy fragment',
|
||||
'chat.snippetAutocomplete.empty': 'Nie znaleziono fragmentów',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ nawigacja • Enter wybierz • Esc zamknij',
|
||||
|
||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
||||
"settings.notifications.page.toast.backgroundDisabled": "Notificações em segundo plano desabilitadas",
|
||||
"settings.notifications.page.testNotification.title": "Notificação de teste",
|
||||
"settings.notifications.page.testNotification.body": "Esta é uma notificação de teste do OpenChamber.",
|
||||
"settings.voice.page.section.voiceSetup": "Configurações de voz",
|
||||
"settings.voice.page.section.speechRecognition": "Reconhecimento de voz",
|
||||
"settings.voice.page.field.enableVoiceInput": "Ativar entrada por voz",
|
||||
"settings.voice.page.field.enableVoiceInputAria": "Ativar entrada por voz (ditado)",
|
||||
"settings.voice.page.section.playbackAndSummary": "Reprodução",
|
||||
"settings.voice.page.field.enableVoiceModeAria": "Ativar modo de voz",
|
||||
"settings.voice.page.field.enableVoiceMode": "Ativar modo de voz",
|
||||
"settings.voice.page.field.provider": "Provedor",
|
||||
"settings.voice.page.provider.browser": "Navegador",
|
||||
"settings.voice.page.provider.custom": "Personalizado",
|
||||
"settings.voice.page.provider.say": "Falar",
|
||||
"settings.voice.page.provider.server": "Servidor",
|
||||
"settings.voice.page.provider.local": "Local",
|
||||
"settings.voice.page.tooltip.sttLocal": "Transcrição local no servidor do OpenChamber. Os modelos são baixados automaticamente; não é necessária chave de API.",
|
||||
"settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro, inglês). O modelo é baixado automaticamente; não é necessária chave de API.",
|
||||
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglês)",
|
||||
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeus)",
|
||||
"settings.voice.page.stt.model.whisperBase": "Whisper base (multilíngue)",
|
||||
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (multilíngue)",
|
||||
"settings.voice.page.stt.badge.bestForEnglish": "Melhor para inglês",
|
||||
"settings.voice.page.stt.badge.bestForMultilingual": "Melhor multilíngue",
|
||||
"settings.voice.page.stt.meta.accuracy": "Precisão",
|
||||
"settings.voice.page.stt.meta.speed": "Velocidade",
|
||||
"settings.voice.page.stt.modelInstalled": "Modelo instalado",
|
||||
"settings.voice.page.stt.modelDownloading": "Baixando modelo...",
|
||||
"settings.voice.page.stt.modelDownloadingProgress": "Baixando modelo... {percent}%",
|
||||
"settings.voice.page.stt.modelNotInstalled": "Modelo não baixado",
|
||||
"settings.voice.page.stt.modelDownload": "Baixar",
|
||||
"settings.voice.page.stt.modelDelete": "Excluir modelo",
|
||||
"settings.voice.page.stt.modelRetry": "Tentar novamente",
|
||||
"settings.voice.page.provider.wasm": "Local",
|
||||
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
|
||||
"settings.voice.page.stt.wasmLoaded": "Modelo carregado",
|
||||
|
||||
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'diffView.reviewDialog.actions.starting': 'Iniciando...',
|
||||
'diffView.reviewDialog.toast.noSessionDirectory': 'O diretório da sessão está indisponível',
|
||||
'diffView.reviewDialog.toast.startFailed': 'Falha ao iniciar o fluxo de revisão',
|
||||
'chat.dictation.start': 'Iniciar ditado',
|
||||
'chat.dictation.overlayAria': 'Ditado',
|
||||
'chat.dictation.downloadingModel': 'Baixando o modelo de fala...',
|
||||
'chat.dictation.downloadingModelProgress': 'Baixando o modelo de fala... {percent}%',
|
||||
'chat.dictation.listening': 'Ouvindo...',
|
||||
'chat.dictation.processing': 'Transcrevendo...',
|
||||
'chat.dictation.failed': 'Falha na transcrição',
|
||||
'chat.dictation.cancel': 'Descartar ditado',
|
||||
'chat.dictation.insert': 'Inserir transcrição',
|
||||
'chat.dictation.insertAndSend': 'Inserir e enviar',
|
||||
'chat.dictation.retry': 'Tentar transcrever novamente',
|
||||
'chat.dictation.discard': 'Descartar gravação',
|
||||
'chat.history.loadOlder': 'Carregar mensagens anteriores',
|
||||
'chat.autoReview.title': 'O ciclo de revisão de código está em andamento',
|
||||
'chat.autoReview.status.waitingForReviewer': 'Aguardando o revisor',
|
||||
|
||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
||||
"settings.notifications.page.toast.backgroundDisabled": "Фонові сповіщення вимкнено",
|
||||
"settings.notifications.page.testNotification.title": "Тестове сповіщення",
|
||||
"settings.notifications.page.testNotification.body": "Це тестове сповіщення від OpenChamber.",
|
||||
"settings.voice.page.section.voiceSetup": "Налаштування голосу",
|
||||
"settings.voice.page.section.speechRecognition": "Розпізнавання мовлення",
|
||||
"settings.voice.page.field.enableVoiceInput": "Увімкнути голосовий ввід",
|
||||
"settings.voice.page.field.enableVoiceInputAria": "Увімкнути голосовий ввід (диктування)",
|
||||
"settings.voice.page.section.playbackAndSummary": "Відтворення",
|
||||
"settings.voice.page.field.enableVoiceModeAria": "Увімкнути голосовий режим",
|
||||
"settings.voice.page.field.enableVoiceMode": "Увімкнути голосовий режим",
|
||||
"settings.voice.page.field.provider": "Провайдер",
|
||||
"settings.voice.page.provider.browser": "Браузер",
|
||||
"settings.voice.page.provider.custom": "Власний",
|
||||
"settings.voice.page.provider.say": "Say",
|
||||
"settings.voice.page.provider.server": "Сервер",
|
||||
"settings.voice.page.provider.local": "Локальний",
|
||||
"settings.voice.page.tooltip.sttLocal": "Локальна розшифровка на сервері OpenChamber. Моделі завантажуються автоматично; ключ API не потрібен.",
|
||||
"settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro, англійська). Модель завантажується автоматично; ключ API не потрібен.",
|
||||
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (англійська)",
|
||||
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 європейських мов)",
|
||||
"settings.voice.page.stt.model.whisperBase": "Whisper base (мультимовна)",
|
||||
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (мультимовна)",
|
||||
"settings.voice.page.stt.badge.bestForEnglish": "Найкраща для англійської",
|
||||
"settings.voice.page.stt.badge.bestForMultilingual": "Найкраща мультимовна",
|
||||
"settings.voice.page.stt.meta.accuracy": "Точність",
|
||||
"settings.voice.page.stt.meta.speed": "Швидкість",
|
||||
"settings.voice.page.stt.modelInstalled": "Модель встановлено",
|
||||
"settings.voice.page.stt.modelDownloading": "Завантаження моделі...",
|
||||
"settings.voice.page.stt.modelDownloadingProgress": "Завантаження моделі... {percent}%",
|
||||
"settings.voice.page.stt.modelNotInstalled": "Модель не завантажено",
|
||||
"settings.voice.page.stt.modelDownload": "Завантажити",
|
||||
"settings.voice.page.stt.modelDelete": "Видалити модель",
|
||||
"settings.voice.page.stt.modelRetry": "Повторити",
|
||||
"settings.voice.page.provider.wasm": "Локально",
|
||||
"settings.voice.page.stt.wasmModel": "Модель Whisper",
|
||||
"settings.voice.page.stt.wasmLoaded": "Модель завантажено",
|
||||
|
||||
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'diffView.reviewDialog.actions.starting': 'Запуск...',
|
||||
'diffView.reviewDialog.toast.noSessionDirectory': 'Директорія сесії недоступна',
|
||||
'diffView.reviewDialog.toast.startFailed': 'Не вдалося запустити review flow',
|
||||
'chat.dictation.start': 'Почати диктування',
|
||||
'chat.dictation.overlayAria': 'Диктування',
|
||||
'chat.dictation.downloadingModel': 'Завантаження мовної моделі...',
|
||||
'chat.dictation.downloadingModelProgress': 'Завантаження мовної моделі... {percent}%',
|
||||
'chat.dictation.listening': 'Слухаю...',
|
||||
'chat.dictation.processing': 'Розшифровка...',
|
||||
'chat.dictation.failed': 'Не вдалося розшифрувати запис',
|
||||
'chat.dictation.cancel': 'Відхилити диктування',
|
||||
'chat.dictation.insert': 'Вставити текст',
|
||||
'chat.dictation.insertAndSend': 'Вставити й надіслати',
|
||||
'chat.dictation.retry': 'Повторити розшифровку',
|
||||
'chat.dictation.discard': 'Відхилити запис',
|
||||
'chat.history.loadOlder': 'Завантажити ще',
|
||||
'chat.autoReview.title': 'Цикл код-ревʼю триває',
|
||||
'chat.autoReview.status.waitingForReviewer': 'Очікуємо ревʼювера',
|
||||
|
||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
||||
'settings.notifications.page.toast.backgroundDisabled': '后台通知已关闭',
|
||||
'settings.notifications.page.testNotification.title': '测试通知',
|
||||
'settings.notifications.page.testNotification.body': '这是来自 OpenChamber 的测试通知。',
|
||||
'settings.voice.page.section.voiceSetup': '语音设置',
|
||||
'settings.voice.page.section.speechRecognition': '语音识别',
|
||||
'settings.voice.page.field.enableVoiceInput': '启用语音输入',
|
||||
'settings.voice.page.field.enableVoiceInputAria': '启用语音输入(听写)',
|
||||
'settings.voice.page.section.playbackAndSummary': '播放',
|
||||
'settings.voice.page.field.enableVoiceModeAria': '启用语音模式',
|
||||
'settings.voice.page.field.enableVoiceMode': '启用语音模式',
|
||||
'settings.voice.page.field.provider': '提供方',
|
||||
'settings.voice.page.provider.browser': '浏览器',
|
||||
'settings.voice.page.provider.custom': '自定义',
|
||||
'settings.voice.page.provider.say': 'Say',
|
||||
'settings.voice.page.provider.server': '服务器',
|
||||
'settings.voice.page.provider.local': '本地',
|
||||
'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 服务器上本地转写。模型自动下载,无需 API 密钥。',
|
||||
'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音(Kokoro,英语)。模型自动下载,无需 API 密钥。',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英语)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(25 种欧洲语言)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多语言)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多语言)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': '英语最佳',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': '多语言最佳',
|
||||
'settings.voice.page.stt.meta.accuracy': '准确度',
|
||||
'settings.voice.page.stt.meta.speed': '速度',
|
||||
'settings.voice.page.stt.modelInstalled': '模型已安装',
|
||||
'settings.voice.page.stt.modelDownloading': '正在下载模型...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': '正在下载模型... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': '模型未下载',
|
||||
'settings.voice.page.stt.modelDownload': '下载',
|
||||
'settings.voice.page.stt.modelDelete': '删除模型',
|
||||
'settings.voice.page.stt.modelRetry': '重试',
|
||||
'settings.voice.page.provider.wasm': '本地',
|
||||
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
|
||||
'settings.voice.page.stt.wasmLoaded': '模型已加载',
|
||||
|
||||
@@ -1839,6 +1839,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'chat.chatInput.placeholder.chat': '@ 用于文件/智能体;/ 用于命令和技能;! 用于 shell;# 用于代码片段',
|
||||
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 辅助',
|
||||
'chat.chatInput.placeholder.selectSession': '选择或创建会话以开始聊天',
|
||||
'chat.dictation.start': '开始语音输入',
|
||||
'chat.dictation.overlayAria': '语音输入',
|
||||
'chat.dictation.downloadingModel': '正在下载语音模型...',
|
||||
'chat.dictation.downloadingModelProgress': '正在下载语音模型... {percent}%',
|
||||
'chat.dictation.listening': '正在聆听...',
|
||||
'chat.dictation.processing': '正在转写...',
|
||||
'chat.dictation.failed': '转写失败',
|
||||
'chat.dictation.cancel': '放弃语音输入',
|
||||
'chat.dictation.insert': '插入文本',
|
||||
'chat.dictation.insertAndSend': '插入并发送',
|
||||
'chat.dictation.retry': '重试转写',
|
||||
'chat.dictation.discard': '放弃录音',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ 新建代码片段',
|
||||
'chat.snippetAutocomplete.empty': '未找到代码片段',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ 导航 • Enter 选择 • Esc 关闭',
|
||||
|
||||
@@ -1401,16 +1401,33 @@
|
||||
'settings.notifications.page.toast.backgroundDisabled': '背景通知已關閉',
|
||||
'settings.notifications.page.testNotification.title': '測試通知',
|
||||
'settings.notifications.page.testNotification.body': '這是來自 OpenChamber 的測試通知。',
|
||||
'settings.voice.page.section.voiceSetup': '語音設定',
|
||||
'settings.voice.page.section.speechRecognition': '語音辨識',
|
||||
'settings.voice.page.field.enableVoiceInput': '啟用語音輸入',
|
||||
'settings.voice.page.field.enableVoiceInputAria': '啟用語音輸入(聽寫)',
|
||||
'settings.voice.page.section.playbackAndSummary': '播放',
|
||||
'settings.voice.page.field.enableVoiceModeAria': '啟用語音模式',
|
||||
'settings.voice.page.field.enableVoiceMode': '啟用語音模式',
|
||||
'settings.voice.page.field.provider': '提供方',
|
||||
'settings.voice.page.provider.browser': '瀏覽器',
|
||||
'settings.voice.page.provider.custom': '自訂',
|
||||
'settings.voice.page.provider.say': 'Say',
|
||||
'settings.voice.page.provider.server': '伺服器',
|
||||
'settings.voice.page.provider.local': '本機',
|
||||
'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 伺服器上本機轉寫。模型會自動下載,無需 API 金鑰。',
|
||||
'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音(Kokoro,英文)。模型會自動下載,無需 API 金鑰。',
|
||||
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英文)',
|
||||
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(25 種歐洲語言)',
|
||||
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多語言)',
|
||||
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多語言)',
|
||||
'settings.voice.page.stt.badge.bestForEnglish': '英文最佳',
|
||||
'settings.voice.page.stt.badge.bestForMultilingual': '多語言最佳',
|
||||
'settings.voice.page.stt.meta.accuracy': '準確度',
|
||||
'settings.voice.page.stt.meta.speed': '速度',
|
||||
'settings.voice.page.stt.modelInstalled': '模型已安裝',
|
||||
'settings.voice.page.stt.modelDownloading': '正在下載模型...',
|
||||
'settings.voice.page.stt.modelDownloadingProgress': '正在下載模型... {percent}%',
|
||||
'settings.voice.page.stt.modelNotInstalled': '模型未下載',
|
||||
'settings.voice.page.stt.modelDownload': '下載',
|
||||
'settings.voice.page.stt.modelDelete': '刪除模型',
|
||||
'settings.voice.page.stt.modelRetry': '重試',
|
||||
'settings.voice.page.provider.wasm': '本機',
|
||||
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
|
||||
'settings.voice.page.stt.wasmLoaded': '模型已載入',
|
||||
|
||||
@@ -1843,6 +1843,18 @@ export const dict: Record<I18nKey, string> = {
|
||||
'chat.chatInput.placeholder.chat': '@ 用於檔案/Agent;/ 用於命令;! 用於 shell;# 用於程式片段',
|
||||
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 輔助',
|
||||
'chat.chatInput.placeholder.selectSession': '選擇或建立會話以開始聊天',
|
||||
'chat.dictation.start': '開始語音輸入',
|
||||
'chat.dictation.overlayAria': '語音輸入',
|
||||
'chat.dictation.downloadingModel': '正在下載語音模型...',
|
||||
'chat.dictation.downloadingModelProgress': '正在下載語音模型... {percent}%',
|
||||
'chat.dictation.listening': '正在聆聽...',
|
||||
'chat.dictation.processing': '正在轉寫...',
|
||||
'chat.dictation.failed': '轉寫失敗',
|
||||
'chat.dictation.cancel': '放棄語音輸入',
|
||||
'chat.dictation.insert': '插入文字',
|
||||
'chat.dictation.insertAndSend': '插入並傳送',
|
||||
'chat.dictation.retry': '重試轉寫',
|
||||
'chat.dictation.discard': '放棄錄音',
|
||||
'chat.snippetAutocomplete.action.addNew': '+ 新建程式片段',
|
||||
'chat.snippetAutocomplete.empty': '未找到程式片段',
|
||||
'chat.snippetAutocomplete.footer': '↑↓ 導航 • Enter 選擇 • Esc 關閉',
|
||||
|
||||
@@ -113,7 +113,10 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
|
||||
localStorage.removeItem('opencode-update-toast-dismissed-version');
|
||||
}
|
||||
}
|
||||
if (settings.sttProvider === 'browser' || settings.sttProvider === 'server') {
|
||||
if (typeof settings.dictationEnabled === 'boolean') {
|
||||
localStorage.setItem('dictationEnabled', String(settings.dictationEnabled));
|
||||
}
|
||||
if (settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') {
|
||||
localStorage.setItem('sttProvider', settings.sttProvider);
|
||||
}
|
||||
if (typeof settings.sttServerUrl === 'string') {
|
||||
@@ -122,18 +125,12 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
|
||||
if (typeof settings.sttModel === 'string') {
|
||||
localStorage.setItem('sttModel', settings.sttModel);
|
||||
}
|
||||
if (typeof settings.sttLocalModel === 'string') {
|
||||
localStorage.setItem('sttLocalModel', settings.sttLocalModel);
|
||||
}
|
||||
if (typeof settings.sttLanguage === 'string') {
|
||||
localStorage.setItem('sttLanguage', settings.sttLanguage);
|
||||
}
|
||||
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb)) {
|
||||
localStorage.setItem('sttSilenceThresholdDb', String(settings.sttSilenceThresholdDb));
|
||||
}
|
||||
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs)) {
|
||||
localStorage.setItem('sttSilenceHoldMs', String(settings.sttSilenceHoldMs));
|
||||
}
|
||||
if (typeof settings.sttTranscribeOnStop === 'boolean') {
|
||||
localStorage.setItem('sttTranscribeOnStop', String(settings.sttTranscribeOnStop));
|
||||
}
|
||||
};
|
||||
|
||||
const dispatchSettingsSynced = (settings: DesktopSettings): void => {
|
||||
@@ -614,7 +611,10 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
|
||||
}
|
||||
if (configStoreApi && configStore) {
|
||||
const nextConfigState: Partial<typeof configStore> = {};
|
||||
if ((settings.sttProvider === 'browser' || settings.sttProvider === 'server') && settings.sttProvider !== configStore.sttProvider) {
|
||||
if (typeof settings.dictationEnabled === 'boolean' && settings.dictationEnabled !== configStore.dictationEnabled) {
|
||||
nextConfigState.dictationEnabled = settings.dictationEnabled;
|
||||
}
|
||||
if ((settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') && settings.sttProvider !== configStore.sttProvider) {
|
||||
nextConfigState.sttProvider = settings.sttProvider;
|
||||
}
|
||||
if (typeof settings.sttServerUrl === 'string' && settings.sttServerUrl !== configStore.sttServerUrl) {
|
||||
@@ -623,18 +623,12 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
|
||||
if (typeof settings.sttModel === 'string' && settings.sttModel !== configStore.sttModel) {
|
||||
nextConfigState.sttModel = settings.sttModel;
|
||||
}
|
||||
if (typeof settings.sttLocalModel === 'string' && settings.sttLocalModel !== configStore.sttLocalModel) {
|
||||
nextConfigState.sttLocalModel = settings.sttLocalModel;
|
||||
}
|
||||
if (typeof settings.sttLanguage === 'string' && settings.sttLanguage !== configStore.sttLanguage) {
|
||||
nextConfigState.sttLanguage = settings.sttLanguage;
|
||||
}
|
||||
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb) && settings.sttSilenceThresholdDb !== configStore.sttSilenceThresholdDb) {
|
||||
nextConfigState.sttSilenceThresholdDb = settings.sttSilenceThresholdDb;
|
||||
}
|
||||
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs) && settings.sttSilenceHoldMs !== configStore.sttSilenceHoldMs) {
|
||||
nextConfigState.sttSilenceHoldMs = settings.sttSilenceHoldMs;
|
||||
}
|
||||
if (typeof settings.sttTranscribeOnStop === 'boolean' && settings.sttTranscribeOnStop !== configStore.sttTranscribeOnStop) {
|
||||
nextConfigState.sttTranscribeOnStop = settings.sttTranscribeOnStop;
|
||||
}
|
||||
if (Object.keys(nextConfigState).length > 0) {
|
||||
configStoreApi.setState(nextConfigState);
|
||||
}
|
||||
@@ -1200,8 +1194,16 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
|
||||
if (typeof candidate.responseStyleCustomInstructions === 'string') {
|
||||
result.responseStyleCustomInstructions = candidate.responseStyleCustomInstructions;
|
||||
}
|
||||
if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'server') {
|
||||
if (typeof candidate.dictationEnabled === 'boolean') {
|
||||
result.dictationEnabled = candidate.dictationEnabled;
|
||||
}
|
||||
if (candidate.sttProvider === 'local' || candidate.sttProvider === 'openai-compatible') {
|
||||
result.sttProvider = candidate.sttProvider;
|
||||
} else if (candidate.sttProvider === 'server') {
|
||||
// Legacy provider migration: 'server' was the OpenAI-compatible endpoint.
|
||||
result.sttProvider = 'openai-compatible';
|
||||
} else if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'wasm') {
|
||||
result.sttProvider = 'local';
|
||||
}
|
||||
if (typeof candidate.sttServerUrl === 'string') {
|
||||
result.sttServerUrl = candidate.sttServerUrl.trim();
|
||||
@@ -1209,18 +1211,12 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
|
||||
if (typeof candidate.sttModel === 'string') {
|
||||
result.sttModel = candidate.sttModel.trim();
|
||||
}
|
||||
if (typeof candidate.sttLocalModel === 'string') {
|
||||
result.sttLocalModel = candidate.sttLocalModel.trim();
|
||||
}
|
||||
if (typeof candidate.sttLanguage === 'string') {
|
||||
result.sttLanguage = candidate.sttLanguage.trim();
|
||||
}
|
||||
if (typeof candidate.sttSilenceThresholdDb === 'number' && Number.isFinite(candidate.sttSilenceThresholdDb)) {
|
||||
result.sttSilenceThresholdDb = candidate.sttSilenceThresholdDb;
|
||||
}
|
||||
if (typeof candidate.sttSilenceHoldMs === 'number' && Number.isFinite(candidate.sttSilenceHoldMs)) {
|
||||
result.sttSilenceHoldMs = candidate.sttSilenceHoldMs;
|
||||
}
|
||||
if (typeof candidate.sttTranscribeOnStop === 'boolean') {
|
||||
result.sttTranscribeOnStop = candidate.sttTranscribeOnStop;
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
@@ -678,22 +678,16 @@ const SETTINGS_SEARCH_ITEMS: readonly SettingsSearchItem[] = [
|
||||
keywords: ['keyboard', 'hotkeys', 'bindings'],
|
||||
},
|
||||
{
|
||||
id: 'voice.voice-setup',
|
||||
id: 'voice.playback',
|
||||
page: 'voice',
|
||||
titleKey: 'settings.voice.page.section.voiceSetup',
|
||||
keywords: ['tts', 'voice mode', 'provider', 'speech rate', 'speech pitch', 'speech volume', 'language'],
|
||||
titleKey: 'settings.voice.page.section.playbackAndSummary',
|
||||
keywords: ['tts', 'read aloud', 'voice', 'provider', 'speech rate', 'speech pitch', 'speech volume', 'tts input mode', 'markdown'],
|
||||
},
|
||||
{
|
||||
id: 'voice.speech-recognition',
|
||||
page: 'voice',
|
||||
titleKey: 'settings.voice.page.section.speechRecognition',
|
||||
keywords: ['stt', 'transcribe', 'whisper', 'microphone', 'silence threshold'],
|
||||
},
|
||||
{
|
||||
id: 'voice.playback',
|
||||
page: 'voice',
|
||||
titleKey: 'settings.voice.page.section.playbackAndSummary',
|
||||
keywords: ['read aloud', 'tts input mode', 'summary', 'markdown'],
|
||||
keywords: ['stt', 'dictation', 'voice input', 'transcribe', 'whisper', 'parakeet', 'microphone'],
|
||||
},
|
||||
{
|
||||
id: 'tunnel.provider',
|
||||
|
||||
@@ -327,6 +327,13 @@ const SHORTCUT_ACTIONS: ReadonlyArray<ShortcutAction> = [
|
||||
description: 'Toggle focus mode for the chat input',
|
||||
customizable: true,
|
||||
},
|
||||
{
|
||||
id: 'toggle_dictation',
|
||||
defaultCombo: 'mod+alt+v',
|
||||
label: 'Voice input',
|
||||
description: 'Start dictation; press again to confirm and insert the transcript',
|
||||
customizable: true,
|
||||
},
|
||||
{
|
||||
id: 'abort_run',
|
||||
defaultCombo: 'escape',
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
/**
|
||||
* Audio Stream Service
|
||||
*
|
||||
* Captures microphone audio using MediaRecorder, detects utterance boundaries
|
||||
* via an AnalyserNode-based silence detector (VAD), then POSTs each utterance
|
||||
* as a raw audio blob to the OpenChamber server's /api/stt/transcribe endpoint.
|
||||
*
|
||||
* Mimics the BrowserVoiceService.startListening interface so useBrowserVoice
|
||||
* can swap providers without changing its internal logic.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* audioStreamService.configure({ baseURL: 'http://localhost:8001/v1', model: 'whisper-1' });
|
||||
* audioStreamService.startListening('en', (text, isFinal) => {
|
||||
* if (isFinal) console.log('transcript:', text);
|
||||
* });
|
||||
* audioStreamService.stopListening();
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
|
||||
type SpeechResultCallback = (text: string, isFinal: boolean) => void;
|
||||
type ErrorCallback = (error: string) => void;
|
||||
|
||||
interface AudioStreamConfig {
|
||||
/** Base URL of the OpenAI-compatible STT server (e.g. http://localhost:8001/v1) */
|
||||
baseURL: string;
|
||||
/** Whisper-compatible model name */
|
||||
model: string;
|
||||
/** Optional BCP-47 language hint (e.g. 'en'). Empty string = auto-detect. */
|
||||
language?: string;
|
||||
/**
|
||||
* Silence threshold in dB below which audio is considered silence.
|
||||
* Lower (more negative) = only very quiet audio counts as silence.
|
||||
* Default: -45
|
||||
*/
|
||||
silenceThresholdDb?: number;
|
||||
/**
|
||||
* How long continuous silence must last (ms) before the utterance is finalised.
|
||||
* Default: 1500
|
||||
*/
|
||||
silenceHoldMs?: number;
|
||||
/** Optional API key for the STT server. */
|
||||
apiKey?: string;
|
||||
}
|
||||
|
||||
// How often (ms) the VAD samples the analyser
|
||||
const VAD_POLL_MS = 80;
|
||||
// Minimum audio duration (ms) to bother uploading (avoids blank clips)
|
||||
const MIN_UTTERANCE_MS = 300;
|
||||
|
||||
class AudioStreamService {
|
||||
private stream: MediaStream | null = null;
|
||||
private mediaRecorder: MediaRecorder | null = null;
|
||||
private audioContext: AudioContext | null = null;
|
||||
private analyser: AnalyserNode | null = null;
|
||||
private vadTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private chunks: Blob[] = [];
|
||||
private recordingStartMs = 0;
|
||||
private isActive = false;
|
||||
private isSpeaking = false;
|
||||
private silenceSince: number | null = null;
|
||||
private onResult: SpeechResultCallback | null = null;
|
||||
private onError: ErrorCallback | null = null;
|
||||
private finishResolver: (() => void) | null = null;
|
||||
private lang = 'en';
|
||||
|
||||
// Configurable parameters
|
||||
private cfg: Required<AudioStreamConfig> = {
|
||||
baseURL: '',
|
||||
model: 'deepdml/faster-whisper-large-v3-turbo-ct2',
|
||||
language: '',
|
||||
silenceThresholdDb: -45,
|
||||
silenceHoldMs: 1500,
|
||||
apiKey: '',
|
||||
};
|
||||
|
||||
/** Update service configuration. Can be called before or after startListening. */
|
||||
configure(config: AudioStreamConfig): void {
|
||||
this.cfg = {
|
||||
silenceThresholdDb: -45,
|
||||
silenceHoldMs: 1500,
|
||||
language: '',
|
||||
apiKey: '',
|
||||
...config,
|
||||
};
|
||||
this.cfg.apiKey = config.apiKey ?? '';
|
||||
}
|
||||
|
||||
/** Whether the browser supports the required APIs. */
|
||||
isSupported(): boolean {
|
||||
return (
|
||||
typeof window !== 'undefined' &&
|
||||
typeof navigator !== 'undefined' &&
|
||||
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
|
||||
typeof window.MediaRecorder !== 'undefined' &&
|
||||
typeof window.AudioContext !== 'undefined'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start listening. Requests microphone access if not already held.
|
||||
* Calls onResult(text, true) for each completed utterance.
|
||||
*/
|
||||
async startListening(
|
||||
lang: string,
|
||||
onResult: SpeechResultCallback,
|
||||
onError?: ErrorCallback
|
||||
): Promise<void> {
|
||||
if (this.isActive) {
|
||||
this.stopListening();
|
||||
}
|
||||
|
||||
this.lang = lang;
|
||||
this.onResult = onResult;
|
||||
this.onError = onError ?? null;
|
||||
this.isActive = true;
|
||||
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
||||
} catch (err) {
|
||||
this.isActive = false;
|
||||
const msg = err instanceof Error ? err.message : 'Microphone access denied';
|
||||
onError?.(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
this._setupAudioContext();
|
||||
this._startRecorder();
|
||||
this._startVAD();
|
||||
}
|
||||
|
||||
/** Stop listening and clean up all resources. */
|
||||
stopListening(): void {
|
||||
this._stopVAD();
|
||||
this._stopRecorder();
|
||||
this._cleanupAfterStop(true);
|
||||
}
|
||||
|
||||
async finishListening(): Promise<void> {
|
||||
if (!this.isActive) return;
|
||||
|
||||
this._stopVAD();
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
|
||||
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
|
||||
this._cleanupAfterStop(true);
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
this.finishResolver = resolve;
|
||||
this._finaliseUtterance(false);
|
||||
});
|
||||
|
||||
this._cleanupAfterStop(true);
|
||||
}
|
||||
|
||||
/** Whether currently listening. */
|
||||
getIsListening(): boolean {
|
||||
return this.isActive;
|
||||
}
|
||||
|
||||
// ── Private helpers ──────────────────────────────────────────────────────
|
||||
|
||||
private _setupAudioContext(): void {
|
||||
if (!this.stream) return;
|
||||
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
|
||||
this.audioContext = new AudioContextClass();
|
||||
const source = this.audioContext.createMediaStreamSource(this.stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 512;
|
||||
source.connect(this.analyser);
|
||||
}
|
||||
|
||||
private _teardownAudioContext(): void {
|
||||
try {
|
||||
this.audioContext?.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
}
|
||||
|
||||
private _startRecorder(): void {
|
||||
if (!this.stream) return;
|
||||
|
||||
const mimeType = this._pickMimeType();
|
||||
const options: MediaRecorderOptions = {};
|
||||
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
|
||||
options.mimeType = mimeType;
|
||||
}
|
||||
|
||||
this.mediaRecorder = new MediaRecorder(this.stream, options);
|
||||
this.chunks = [];
|
||||
this.recordingStartMs = Date.now();
|
||||
|
||||
this.mediaRecorder.ondataavailable = (e) => {
|
||||
if (e.data && e.data.size > 0) {
|
||||
this.chunks.push(e.data);
|
||||
}
|
||||
};
|
||||
|
||||
this.mediaRecorder.onstop = () => {
|
||||
const blobs = this.chunks.splice(0);
|
||||
const durationMs = Date.now() - this.recordingStartMs;
|
||||
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
|
||||
this.finishResolver?.();
|
||||
this.finishResolver = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const mType = blobs[0].type || mimeType || 'audio/webm';
|
||||
const blob = new Blob(blobs, { type: mType });
|
||||
void this._upload(blob, mType).finally(() => {
|
||||
this.finishResolver?.();
|
||||
this.finishResolver = null;
|
||||
});
|
||||
};
|
||||
|
||||
// Collect data every 250 ms so we don't lose the tail on stop()
|
||||
this.mediaRecorder.start(250);
|
||||
}
|
||||
|
||||
private _stopRecorder(): void {
|
||||
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
|
||||
try {
|
||||
this.mediaRecorder.stop();
|
||||
} catch {
|
||||
this.finishResolver?.();
|
||||
this.finishResolver = null;
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
this.mediaRecorder = null;
|
||||
}
|
||||
|
||||
private _releaseStream(): void {
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
}
|
||||
|
||||
private _startVAD(): void {
|
||||
this._stopVAD();
|
||||
this.silenceSince = null;
|
||||
this.isSpeaking = false;
|
||||
|
||||
this.vadTimer = setInterval(() => {
|
||||
if (!this.isActive || !this.analyser) return;
|
||||
const db = this._getRmsDb();
|
||||
const isSilent = db < this.cfg.silenceThresholdDb;
|
||||
|
||||
if (!isSilent) {
|
||||
// Audio detected
|
||||
this.silenceSince = null;
|
||||
if (!this.isSpeaking) {
|
||||
this.isSpeaking = true;
|
||||
// Restart recorder to capture from the start of speech
|
||||
if (this.mediaRecorder?.state === 'recording') {
|
||||
this.recordingStartMs = Date.now();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Silence detected
|
||||
if (this.isSpeaking) {
|
||||
if (this.silenceSince === null) {
|
||||
this.silenceSince = Date.now();
|
||||
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
|
||||
// End of utterance — stop recorder (triggers onstop → upload)
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
this._finaliseUtterance(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, VAD_POLL_MS);
|
||||
}
|
||||
|
||||
private _stopVAD(): void {
|
||||
if (this.vadTimer !== null) {
|
||||
clearInterval(this.vadTimer);
|
||||
this.vadTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private _cleanupAfterStop(clearChunks: boolean): void {
|
||||
const pendingResolver = this.finishResolver;
|
||||
this.isActive = false;
|
||||
this.finishResolver = null;
|
||||
this.mediaRecorder = null;
|
||||
this._teardownAudioContext();
|
||||
this._releaseStream();
|
||||
if (clearChunks) {
|
||||
this.chunks = [];
|
||||
}
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
this.onResult = null;
|
||||
this.onError = null;
|
||||
pendingResolver?.();
|
||||
}
|
||||
|
||||
/** Stop the current recorder to flush the utterance, optionally restarting for the next one. */
|
||||
private _finaliseUtterance(restart: boolean): void {
|
||||
if (!this.isActive) return;
|
||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
||||
this.mediaRecorder.stop();
|
||||
}
|
||||
if (!restart) return;
|
||||
|
||||
// Restart recorder for the next utterance after a short delay
|
||||
// (MediaRecorder.onstop fires asynchronously; we wait for it to complete)
|
||||
setTimeout(() => {
|
||||
if (this.isActive && this.stream) {
|
||||
this._startRecorder();
|
||||
}
|
||||
}, 100);
|
||||
}
|
||||
|
||||
/** Compute RMS of current analyser frame in dBFS. */
|
||||
private _getRmsDb(): number {
|
||||
if (!this.analyser) return -Infinity;
|
||||
const buf = new Float32Array(this.analyser.fftSize);
|
||||
this.analyser.getFloatTimeDomainData(buf);
|
||||
let sumSq = 0;
|
||||
for (const s of buf) sumSq += s * s;
|
||||
const rms = Math.sqrt(sumSq / buf.length);
|
||||
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
|
||||
}
|
||||
|
||||
/** POST utterance blob to server, call onResult with transcript. */
|
||||
private async _upload(blob: Blob, mimeType: string): Promise<void> {
|
||||
if (!this.onResult) return;
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': mimeType,
|
||||
'X-Base-URL': this.cfg.baseURL,
|
||||
'X-Model': this.cfg.model,
|
||||
};
|
||||
if (this.cfg.apiKey) {
|
||||
headers['Authorization'] = `Bearer ${this.cfg.apiKey}`;
|
||||
}
|
||||
if (this.cfg.language) {
|
||||
headers['X-Language'] = this.cfg.language;
|
||||
} else if (this.lang && this.lang !== 'auto') {
|
||||
// Use BCP-47 base language code (e.g. 'en' from 'en-US')
|
||||
const baseLang = this.lang.split('-')[0];
|
||||
headers['X-Language'] = baseLang;
|
||||
}
|
||||
|
||||
const response = await runtimeFetch('/api/stt/transcribe', {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: blob,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errData.error ?? `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const transcript: string = (data.transcript ?? '').trim();
|
||||
if (transcript) {
|
||||
this.onResult(transcript, true);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!this.isActive) return; // Stopped — ignore
|
||||
const msg = err instanceof Error ? err.message : 'Transcription upload failed';
|
||||
console.error('[AudioStreamService] Upload error:', msg);
|
||||
this.onError?.(msg);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pick the best supported MIME type for MediaRecorder. */
|
||||
private _pickMimeType(): string {
|
||||
const candidates = [
|
||||
'audio/webm;codecs=opus',
|
||||
'audio/webm',
|
||||
'audio/ogg;codecs=opus',
|
||||
'audio/ogg',
|
||||
'audio/mp4',
|
||||
];
|
||||
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
|
||||
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
export const audioStreamService = new AudioStreamService();
|
||||
@@ -1,131 +0,0 @@
|
||||
/**
|
||||
* Context formatters for voice-native output
|
||||
* Formats session events (messages, permissions, ready events) into natural language
|
||||
* for the ElevenLabs voice agent to speak aloud.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { formatMessage, formatPermissionRequest } from '@/lib/voice';
|
||||
*
|
||||
* const voiceText = formatMessage({ role: 'assistant', content: 'Hello!' });
|
||||
* // Returns: "Claude Code: Hello!"
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { VOICE_CONFIG } from "./voiceConfig";
|
||||
|
||||
/** Message type for voice formatting */
|
||||
export interface VoiceMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a single message for voice output
|
||||
* - Assistant messages: Code blocks replaced with "[code block]", prefixed with "Claude Code: "
|
||||
* - User messages: Prefixed with "User: "
|
||||
* - Other roles: Returns null (not spoken)
|
||||
*
|
||||
* @param message - The message to format
|
||||
* @returns Formatted text for voice, or null if should not be spoken
|
||||
*/
|
||||
function formatMessage(message: VoiceMessage): string | null {
|
||||
// Handle edge cases
|
||||
if (!message || typeof message.content !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = message.content.trim();
|
||||
if (!content) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (message.role === "assistant") {
|
||||
// Replace code blocks with description (don't read code aloud)
|
||||
const textOnly = content.replace(/```[\s\S]*?```/g, "[code block]");
|
||||
return `Claude Code: ${textOnly}`;
|
||||
}
|
||||
|
||||
if (message.role === "user") {
|
||||
return `User: ${content}`;
|
||||
}
|
||||
|
||||
// Skip system, tool, and other roles for voice
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format multiple new messages for voice output
|
||||
* - Maps messages through formatMessage
|
||||
* - Filters out nulls (unspoken roles)
|
||||
* - Joins with newlines
|
||||
*
|
||||
* @param sessionId - The session ID (for future use/debugging)
|
||||
* @param messages - Array of messages to format
|
||||
* @returns Formatted text for voice, or null if no speakable messages
|
||||
*/
|
||||
export function formatNewMessages(
|
||||
sessionId: string,
|
||||
messages: VoiceMessage[]
|
||||
): string | null {
|
||||
// Handle edge cases
|
||||
if (!Array.isArray(messages) || messages.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log(`[Voice] Formatting ${messages.length} messages for session ${sessionId}`);
|
||||
}
|
||||
|
||||
// Format each message and filter out nulls
|
||||
const formattedMessages = messages
|
||||
.map(formatMessage)
|
||||
.filter((msg): msg is string => msg !== null);
|
||||
|
||||
if (formattedMessages.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return formattedMessages.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a permission request for voice announcement
|
||||
* - Per CONTEXT.md: Only tool name, not arguments (LIMITED_TOOL_CALLS)
|
||||
* - Prompts user to say "allow" or "deny"
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @param requestId - The permission request ID
|
||||
* @param toolName - Name of the tool requesting permission
|
||||
* @param toolArgs - Tool arguments (not included in voice output per config)
|
||||
* @returns Formatted permission request for voice
|
||||
*/
|
||||
export function formatPermissionRequest(
|
||||
sessionId: string,
|
||||
requestId: string,
|
||||
toolName: string,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
toolArgs: unknown
|
||||
): string {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log(`[Voice] Formatting permission request ${requestId} for session ${sessionId}`);
|
||||
}
|
||||
|
||||
// Per VOICE_CONFIG.LIMITED_TOOL_CALLS, we don't include toolArgs in voice output
|
||||
return `Claude Code is requesting permission to use ${toolName}. Say "allow" or "deny".`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a ready event for voice announcement
|
||||
* - Indicates the AI has finished working and is ready for next instruction
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @returns Formatted ready event for voice
|
||||
*/
|
||||
export function formatReadyEvent(sessionId: string): string {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log(`[Voice] Formatting ready event for session ${sessionId}`);
|
||||
}
|
||||
|
||||
return "Claude Code finished working. Ready for next instruction.";
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
/**
|
||||
* Voice module barrel export
|
||||
* Provides a clean import path for voice session hooks.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { voiceHooks } from '@/lib/voice';
|
||||
* ```
|
||||
*/
|
||||
|
||||
// Voice session registry (from voiceSession.ts)
|
||||
export {
|
||||
isVoiceSessionStarted,
|
||||
} from "./voiceSession";
|
||||
|
||||
// Voice hooks for session-to-voice event routing (from voiceHooks.ts)
|
||||
export { voiceHooks } from "./voiceHooks";
|
||||
@@ -6,15 +6,21 @@
|
||||
export function sanitizeForTTS(text: string): string {
|
||||
if (!text) return '';
|
||||
return text
|
||||
// Remove code blocks
|
||||
// Remove fenced code blocks entirely (multi-line code is unreadable),
|
||||
// but keep inline-code CONTENT and only strip the backticks: agents
|
||||
// routinely inline meaningful words ("You are on `main`").
|
||||
.replace(/```[\s\S]*?```/g, '')
|
||||
.replace(/`[^`]*`/g, '')
|
||||
.replace(/`([^`\n]*)`/g, '$1')
|
||||
// Remove markdown formatting
|
||||
.replace(/[*_~#]/g, '')
|
||||
// Remove URLs
|
||||
.replace(/https?:\/\/[^\s]+/g, '')
|
||||
// Remove file paths
|
||||
.replace(/\/[\w\-./]+/g, '')
|
||||
// Remove absolute file paths (leading slash, one or more segments).
|
||||
// Deliberately NOT matching interword slashes: "iOS/Android" and
|
||||
// "origin/main" are speech, not paths.
|
||||
.replace(/(^|\s)\/(?:[\w.-]+\/)*[\w.-]+/g, '$1')
|
||||
// Read remaining interword slashes out loud ("iOS slash Android").
|
||||
.replace(/([\w.])\/([\w.])/g, '$1 slash $2')
|
||||
// Remove shell-like patterns
|
||||
.replace(/^\s*[$#>]\s*/gm, '')
|
||||
// Remove brackets and special chars
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
/**
|
||||
* Static voice context configuration
|
||||
* Controls voice behavior and feature flags for the ElevenLabs voice agent
|
||||
*/
|
||||
export const VOICE_CONFIG = {
|
||||
/** Disable all tool call information from being sent to voice context */
|
||||
DISABLE_TOOL_CALLS: false,
|
||||
|
||||
/** Send only tool names and descriptions, exclude arguments */
|
||||
LIMITED_TOOL_CALLS: true,
|
||||
|
||||
/** Disable permission request forwarding */
|
||||
DISABLE_PERMISSION_REQUESTS: false,
|
||||
|
||||
/** Disable session online/offline notifications */
|
||||
DISABLE_SESSION_STATUS: true,
|
||||
|
||||
/** Disable message forwarding */
|
||||
DISABLE_MESSAGES: false,
|
||||
|
||||
/** Disable session focus notifications */
|
||||
DISABLE_SESSION_FOCUS: false,
|
||||
|
||||
/** Disable ready event notifications */
|
||||
DISABLE_READY_EVENTS: false,
|
||||
|
||||
/** Maximum number of messages to include in session history */
|
||||
MAX_HISTORY_MESSAGES: 50,
|
||||
|
||||
/** Enable debug logging for voice context updates */
|
||||
ENABLE_DEBUG_LOGGING: true,
|
||||
} as const;
|
||||
@@ -1,125 +0,0 @@
|
||||
/**
|
||||
* Voice hooks for session-to-voice event routing
|
||||
* Routes session events (messages, permissions, ready events) to the ElevenLabs
|
||||
* voice agent via contextual updates.
|
||||
*
|
||||
* This module provides hooks that can be called when session events occur,
|
||||
* using the voice session registry from voiceSession.ts.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { voiceHooks } from '@/lib/voice';
|
||||
*
|
||||
* // Route session messages to voice
|
||||
* voiceHooks.onMessages(sessionId, messages);
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { VOICE_CONFIG } from "./voiceConfig";
|
||||
import {
|
||||
formatNewMessages,
|
||||
formatPermissionRequest,
|
||||
formatReadyEvent,
|
||||
type VoiceMessage,
|
||||
} from "./contextFormatters";
|
||||
import { getVoiceSession, isVoiceSessionStarted } from "./voiceSession";
|
||||
|
||||
/**
|
||||
* Report a contextual update to the voice session
|
||||
* Internal helper that checks preconditions and handles errors
|
||||
*
|
||||
* @param update - The text update to send, or null/undefined to skip
|
||||
*/
|
||||
function reportContextualUpdate(update: string | null | undefined): void {
|
||||
// Skip empty/null/undefined updates
|
||||
if (!update || update.trim().length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip if no voice session or not started
|
||||
const voiceSession = getVoiceSession();
|
||||
if (!voiceSession || !isVoiceSessionStarted()) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Skipping contextual update - no active session");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Sending contextual update:", update.substring(0, 100));
|
||||
}
|
||||
voiceSession.sendContextualUpdate(update);
|
||||
} catch (error) {
|
||||
// Log error but don't throw - voice updates shouldn't break the app
|
||||
console.error("[Voice] Failed to send contextual update:", error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Voice hooks - exported functions to route session events to voice
|
||||
*
|
||||
* These hooks should be called when corresponding session events occur.
|
||||
* They respect VOICE_CONFIG feature flags to enable/disable specific
|
||||
* event types.
|
||||
*/
|
||||
export const voiceHooks = {
|
||||
/**
|
||||
* Called when new messages arrive in the session
|
||||
* Formats and sends messages to voice agent (if not disabled)
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @param messages - Array of messages to format and send
|
||||
*/
|
||||
onMessages(sessionId: string, messages: VoiceMessage[]): void {
|
||||
if (VOICE_CONFIG.DISABLE_MESSAGES) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Message forwarding disabled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
reportContextualUpdate(formatNewMessages(sessionId, messages));
|
||||
},
|
||||
|
||||
/**
|
||||
* Called when a permission request is made
|
||||
* Announces the permission request to the voice agent (if not disabled)
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @param requestId - The permission request ID
|
||||
* @param toolName - Name of the tool requesting permission
|
||||
* @param toolArgs - Arguments for the tool (not sent to voice per LIMITED_TOOL_CALLS)
|
||||
*/
|
||||
onPermissionRequested(
|
||||
sessionId: string,
|
||||
requestId: string,
|
||||
toolName: string,
|
||||
toolArgs: unknown
|
||||
): void {
|
||||
if (VOICE_CONFIG.DISABLE_PERMISSION_REQUESTS) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Permission request forwarding disabled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
reportContextualUpdate(
|
||||
formatPermissionRequest(sessionId, requestId, toolName, toolArgs)
|
||||
);
|
||||
},
|
||||
|
||||
/**
|
||||
* Called when the AI is ready for the next instruction
|
||||
* Announces ready state to the voice agent (if not disabled)
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
*/
|
||||
onReady(sessionId: string): void {
|
||||
if (VOICE_CONFIG.DISABLE_READY_EVENTS) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Ready event forwarding disabled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
reportContextualUpdate(formatReadyEvent(sessionId));
|
||||
},
|
||||
};
|
||||
@@ -1,28 +0,0 @@
|
||||
/**
|
||||
* Voice session interface
|
||||
* Used for type safety without importing ReturnType from SDK
|
||||
*/
|
||||
interface VoiceSession {
|
||||
sendContextualUpdate: (text: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Global storage for the active voice session.
|
||||
* Used by voiceHooks to send contextual updates to the voice agent.
|
||||
*/
|
||||
const activeVoiceSession: VoiceSession | null = null;
|
||||
|
||||
/**
|
||||
* Get the currently registered voice session.
|
||||
* Used by voiceHooks to send contextual updates.
|
||||
*/
|
||||
export function getVoiceSession(): VoiceSession | null {
|
||||
return activeVoiceSession;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a voice session is currently active.
|
||||
*/
|
||||
export function isVoiceSessionStarted(): boolean {
|
||||
return activeVoiceSession !== null;
|
||||
}
|
||||
@@ -1,555 +0,0 @@
|
||||
/**
|
||||
* WASM Speech-to-Text Service
|
||||
*
|
||||
* Local Whisper transcription via Transformers.js (ONNX Runtime Web).
|
||||
* Captures microphone audio, detects utterance boundaries via silence-based
|
||||
* VAD, then transcribes each utterance locally — no cloud API required.
|
||||
*
|
||||
* Works in Electron and all modern browsers that support Web Audio API.
|
||||
* First use downloads a Whisper model (~40–166 MB, cached).
|
||||
*/
|
||||
|
||||
export type WasmModelStatus =
|
||||
| { state: 'unloaded' }
|
||||
| { state: 'downloading'; progress: number }
|
||||
| { state: 'loading' }
|
||||
| { state: 'ready' }
|
||||
| { state: 'error'; error: string };
|
||||
|
||||
export interface WasmModelInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
size: string;
|
||||
languages: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export const WASM_MODELS: WasmModelInfo[] = [
|
||||
{
|
||||
id: 'Xenova/whisper-tiny.en',
|
||||
name: 'Whisper Tiny (EN)',
|
||||
size: '~39 MB',
|
||||
languages: 'English',
|
||||
description: 'Fastest, lowest accuracy. Good for quick dictation.',
|
||||
},
|
||||
{
|
||||
id: 'Xenova/whisper-base.en',
|
||||
name: 'Whisper Base (EN)',
|
||||
size: '~73 MB',
|
||||
languages: 'English',
|
||||
description: 'Balanced speed and accuracy. Default for English.',
|
||||
},
|
||||
{
|
||||
id: 'Xenova/whisper-small.en',
|
||||
name: 'Whisper Small (EN)',
|
||||
size: '~166 MB',
|
||||
languages: 'English',
|
||||
description: 'Higher accuracy, slower. Best for noisy environments.',
|
||||
},
|
||||
];
|
||||
|
||||
type SpeechResultCallback = (text: string, isFinal: boolean) => void;
|
||||
type ErrorCallback = (error: string) => void;
|
||||
|
||||
const VAD_POLL_MS = 80;
|
||||
const MIN_UTTERANCE_MS = 300;
|
||||
const WHISPER_SAMPLE_RATE = 16000;
|
||||
|
||||
interface WasmSttConfig {
|
||||
silenceThresholdDb?: number;
|
||||
silenceHoldMs?: number;
|
||||
}
|
||||
|
||||
class WasmSttService {
|
||||
private transcriber: unknown = null;
|
||||
private worker: Worker | null = null;
|
||||
private modelStatus: WasmModelStatus = { state: 'unloaded' };
|
||||
private currentModelId: string | null = null;
|
||||
|
||||
private stream: MediaStream | null = null;
|
||||
private mediaRecorder: MediaRecorder | null = null;
|
||||
private audioContext: AudioContext | null = null;
|
||||
private analyser: AnalyserNode | null = null;
|
||||
private vadTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private chunks: Blob[] = [];
|
||||
private recordingStartMs = 0;
|
||||
private isActive = false;
|
||||
private isSpeaking = false;
|
||||
private silenceSince: number | null = null;
|
||||
private onResult: SpeechResultCallback | null = null;
|
||||
private onError: ErrorCallback | null = null;
|
||||
private finishResolver: (() => void) | null = null;
|
||||
private lang = 'en';
|
||||
|
||||
private cfg: Required<WasmSttConfig> = {
|
||||
silenceThresholdDb: -45,
|
||||
silenceHoldMs: 1500,
|
||||
};
|
||||
|
||||
public onModelStatusChange: ((status: WasmModelStatus) => void) | null = null;
|
||||
|
||||
configure(config: WasmSttConfig): void {
|
||||
this.cfg = { ...this.cfg, ...config };
|
||||
}
|
||||
|
||||
isSupported(): boolean {
|
||||
return (
|
||||
typeof window !== 'undefined' &&
|
||||
typeof navigator !== 'undefined' &&
|
||||
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
|
||||
typeof window.MediaRecorder !== 'undefined' &&
|
||||
typeof window.AudioContext !== 'undefined'
|
||||
);
|
||||
}
|
||||
|
||||
getModelStatus(): WasmModelStatus {
|
||||
return this.modelStatus;
|
||||
}
|
||||
|
||||
getCurrentModelId(): string | null {
|
||||
return this.currentModelId;
|
||||
}
|
||||
|
||||
private setModelStatus(status: WasmModelStatus): void {
|
||||
this.modelStatus = status;
|
||||
this.onModelStatusChange?.(status);
|
||||
}
|
||||
|
||||
async loadModel(modelId: string): Promise<void> {
|
||||
if (this.currentModelId === modelId && this.modelStatus.state === 'ready') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.modelStatus.state === 'downloading' || this.modelStatus.state === 'loading') {
|
||||
return;
|
||||
}
|
||||
|
||||
this._terminateWorker();
|
||||
this.transcriber = null;
|
||||
|
||||
this.setModelStatus({ state: 'downloading', progress: 0 });
|
||||
this.currentModelId = modelId;
|
||||
|
||||
// Try Web Worker first — inference off main thread = no UI freeze.
|
||||
try {
|
||||
const WasmWorkerMod = await import('./wasmSttWorker?worker');
|
||||
const WasmWorker = WasmWorkerMod.default as new () => Worker;
|
||||
this.worker = new WasmWorker();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('Worker init timed out')), 10000);
|
||||
|
||||
this.worker!.onmessage = (e: MessageEvent) => {
|
||||
const data = e.data as { type: string; progress?: number; error?: string; text?: string };
|
||||
if (data.type === 'progress') {
|
||||
this.setModelStatus({ state: 'downloading', progress: data.progress ?? 0 });
|
||||
} else if (data.type === 'loaded') {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
} else if (data.type === 'error') {
|
||||
clearTimeout(timer);
|
||||
reject(new Error(data.error ?? 'Worker load failed'));
|
||||
}
|
||||
};
|
||||
|
||||
this.worker!.onerror = (err) => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error(err.message || 'Worker error'));
|
||||
};
|
||||
|
||||
this.worker!.postMessage({ type: 'load', modelId });
|
||||
});
|
||||
|
||||
this.setModelStatus({ state: 'ready' });
|
||||
return;
|
||||
} catch (err) {
|
||||
console.warn('[WasmStt] Worker failed, using main-thread:', err instanceof Error ? err.message : err);
|
||||
this._terminateWorker();
|
||||
}
|
||||
|
||||
// Fallback: main-thread pipeline (causes brief UI freeze during inference).
|
||||
try {
|
||||
const { pipeline, env } = await import('@xenova/transformers');
|
||||
env.backends.onnx.wasm.numThreads = 1;
|
||||
env.allowLocalModels = false;
|
||||
|
||||
const fileDoneBytes = new Map<string, number>();
|
||||
let totalDone = 0;
|
||||
let totalEstimate = 0;
|
||||
|
||||
this.transcriber = await pipeline('automatic-speech-recognition', modelId, {
|
||||
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
|
||||
if (info.status === 'progress' && info.file) {
|
||||
const prevDone = fileDoneBytes.get(info.file) ?? 0;
|
||||
const currentDone = info.loaded ?? 0;
|
||||
const delta = Math.max(0, currentDone - prevDone);
|
||||
fileDoneBytes.set(info.file, currentDone);
|
||||
totalDone += delta;
|
||||
if (info.total && info.total > totalEstimate) totalEstimate = info.total;
|
||||
const effectiveTotal = Math.max(totalEstimate, totalDone);
|
||||
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
|
||||
this.setModelStatus({ state: 'downloading', progress: pct });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
this.setModelStatus({ state: 'ready' });
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Unknown error loading model';
|
||||
this.setModelStatus({ state: 'error', error: msg });
|
||||
this.transcriber = null;
|
||||
this.currentModelId = null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private _terminateWorker(): void {
|
||||
if (this.worker) {
|
||||
this.worker.terminate();
|
||||
this.worker = null;
|
||||
}
|
||||
}
|
||||
|
||||
async unloadModel(): Promise<void> {
|
||||
this._terminateWorker();
|
||||
this.transcriber = null;
|
||||
this.currentModelId = null;
|
||||
this.setModelStatus({ state: 'unloaded' });
|
||||
}
|
||||
|
||||
async startListening(
|
||||
lang: string,
|
||||
onResult: SpeechResultCallback,
|
||||
onError?: ErrorCallback,
|
||||
): Promise<void> {
|
||||
if (this.isActive) {
|
||||
this.stopListening();
|
||||
}
|
||||
|
||||
if (!this.transcriber && !this.worker) {
|
||||
onError?.('Whisper model not loaded. Select a model in Voice Settings first.');
|
||||
return;
|
||||
}
|
||||
|
||||
this.lang = lang;
|
||||
this.onResult = onResult;
|
||||
this.onError = onError ?? null;
|
||||
this.isActive = true;
|
||||
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
||||
} catch (err) {
|
||||
this.isActive = false;
|
||||
const msg = err instanceof Error ? err.message : 'Microphone access denied';
|
||||
onError?.(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
this._setupAudioContext();
|
||||
this._startRecorder();
|
||||
this._startVAD();
|
||||
}
|
||||
|
||||
stopListening(): void {
|
||||
this._stopVAD();
|
||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
||||
try { this.mediaRecorder.stop(); } catch { /* ignore */ }
|
||||
}
|
||||
this._cleanupAfterStop(true);
|
||||
}
|
||||
|
||||
async finishListening(): Promise<void> {
|
||||
if (!this.isActive) return;
|
||||
|
||||
this._stopVAD();
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
|
||||
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
|
||||
this._cleanupAfterStop(true);
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
this.finishResolver = resolve;
|
||||
this._finaliseUtterance(false);
|
||||
});
|
||||
|
||||
this._cleanupAfterStop(true);
|
||||
}
|
||||
|
||||
getIsListening(): boolean {
|
||||
return this.isActive;
|
||||
}
|
||||
|
||||
// ── Audio capture ────────────────────────────────────────────────────
|
||||
|
||||
private _setupAudioContext(): void {
|
||||
if (!this.stream) return;
|
||||
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
|
||||
this.audioContext = new AudioContextClass();
|
||||
const source = this.audioContext.createMediaStreamSource(this.stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 512;
|
||||
source.connect(this.analyser);
|
||||
}
|
||||
|
||||
private _teardownAudioContext(): void {
|
||||
try { this.audioContext?.close(); } catch { /* ignore */ }
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
}
|
||||
|
||||
private _startRecorder(): void {
|
||||
if (!this.stream) return;
|
||||
const mimeType = this._pickMimeType();
|
||||
const options: MediaRecorderOptions = {};
|
||||
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
|
||||
options.mimeType = mimeType;
|
||||
}
|
||||
this.mediaRecorder = new MediaRecorder(this.stream, options);
|
||||
this.chunks = [];
|
||||
this.recordingStartMs = Date.now();
|
||||
|
||||
this.mediaRecorder.ondataavailable = (e) => {
|
||||
if (e.data && e.data.size > 0) {
|
||||
this.chunks.push(e.data);
|
||||
}
|
||||
};
|
||||
|
||||
this.mediaRecorder.onstop = () => {
|
||||
const blobs = this.chunks.splice(0);
|
||||
const durationMs = Date.now() - this.recordingStartMs;
|
||||
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
|
||||
this.finishResolver?.();
|
||||
this.finishResolver = null;
|
||||
return;
|
||||
}
|
||||
const mType = blobs[0].type || mimeType || 'audio/webm';
|
||||
const blob = new Blob(blobs, { type: mType });
|
||||
void this._transcribe(blob).finally(() => {
|
||||
this.finishResolver?.();
|
||||
this.finishResolver = null;
|
||||
});
|
||||
};
|
||||
|
||||
this.mediaRecorder.start(250);
|
||||
}
|
||||
|
||||
private _releaseStream(): void {
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── VAD ──────────────────────────────────────────────────────────────
|
||||
|
||||
private _startVAD(): void {
|
||||
this._stopVAD();
|
||||
this.silenceSince = null;
|
||||
this.isSpeaking = false;
|
||||
|
||||
this.vadTimer = setInterval(() => {
|
||||
if (!this.isActive || !this.analyser) return;
|
||||
const db = this._getRmsDb();
|
||||
const isSilent = db < this.cfg.silenceThresholdDb;
|
||||
|
||||
if (!isSilent) {
|
||||
this.silenceSince = null;
|
||||
if (!this.isSpeaking) {
|
||||
this.isSpeaking = true;
|
||||
if (this.mediaRecorder?.state === 'recording') {
|
||||
this.recordingStartMs = Date.now();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (this.isSpeaking) {
|
||||
if (this.silenceSince === null) {
|
||||
this.silenceSince = Date.now();
|
||||
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
this._finaliseUtterance(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, VAD_POLL_MS);
|
||||
}
|
||||
|
||||
private _stopVAD(): void {
|
||||
if (this.vadTimer !== null) {
|
||||
clearInterval(this.vadTimer);
|
||||
this.vadTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private _cleanupAfterStop(clearChunks: boolean): void {
|
||||
const pendingResolver = this.finishResolver;
|
||||
this.isActive = false;
|
||||
this.finishResolver = null;
|
||||
this.mediaRecorder = null;
|
||||
this._teardownAudioContext();
|
||||
this._releaseStream();
|
||||
if (clearChunks) this.chunks = [];
|
||||
this.isSpeaking = false;
|
||||
this.silenceSince = null;
|
||||
this.onResult = null;
|
||||
this.onError = null;
|
||||
pendingResolver?.();
|
||||
}
|
||||
|
||||
private _finaliseUtterance(restart: boolean): void {
|
||||
if (!this.isActive) return;
|
||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
||||
this.mediaRecorder.stop();
|
||||
}
|
||||
if (!restart) return;
|
||||
setTimeout(() => {
|
||||
if (this.isActive && this.stream) {
|
||||
this._startRecorder();
|
||||
}
|
||||
}, 100);
|
||||
}
|
||||
|
||||
private _getRmsDb(): number {
|
||||
if (!this.analyser) return -Infinity;
|
||||
const buf = new Float32Array(this.analyser.fftSize);
|
||||
this.analyser.getFloatTimeDomainData(buf);
|
||||
let sumSq = 0;
|
||||
for (const s of buf) sumSq += s * s;
|
||||
const rms = Math.sqrt(sumSq / buf.length);
|
||||
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
|
||||
}
|
||||
|
||||
// ── Transcription ────────────────────────────────────────────────────
|
||||
|
||||
private async _transcribe(blob: Blob): Promise<void> {
|
||||
if (!this.onResult) return;
|
||||
if (!this.transcriber && !this.worker) {
|
||||
this.onError?.('Model not loaded');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const audioData = await this._decodeToFloat32(blob);
|
||||
if (!audioData || audioData.length === 0) {
|
||||
this.onError?.(`Failed to decode audio (${blob.size} bytes)`);
|
||||
return;
|
||||
}
|
||||
|
||||
const langHint = this._resolveLanguageHint();
|
||||
|
||||
// Prefer worker (non-blocking); fall back to main-thread pipeline.
|
||||
const transcript = this.worker
|
||||
? await this._transcribeViaWorker(audioData, langHint)
|
||||
: await this._transcribeMainThread(audioData, langHint);
|
||||
|
||||
if (transcript) {
|
||||
this.onResult(transcript, true);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!this.isActive) return;
|
||||
const msg = err instanceof Error ? err.message : 'Local transcription failed';
|
||||
this.onError?.(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private _transcribeViaWorker(audioData: Float32Array, langHint: string | undefined): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!this.worker) return reject(new Error('Worker gone'));
|
||||
|
||||
const onMessage = (e: MessageEvent) => {
|
||||
const data = e.data as { type: string; error?: string; transcript?: string; text?: string };
|
||||
if (data.type === 'result') {
|
||||
this.worker!.removeEventListener('message', onMessage);
|
||||
resolve(data.transcript ?? '');
|
||||
} else if (data.type === 'log') {
|
||||
console.log('[WasmStt Worker]', data.text);
|
||||
} else if (data.type === 'error') {
|
||||
this.worker!.removeEventListener('message', onMessage);
|
||||
reject(new Error(data.error ?? 'Transcription failed'));
|
||||
}
|
||||
};
|
||||
|
||||
this.worker.addEventListener('message', onMessage);
|
||||
this.worker.postMessage(
|
||||
{ type: 'transcribe', audio: audioData.buffer, language: langHint },
|
||||
[audioData.buffer],
|
||||
);
|
||||
|
||||
setTimeout(() => {
|
||||
this.worker?.removeEventListener('message', onMessage);
|
||||
reject(new Error('Transcription timed out'));
|
||||
}, 30000);
|
||||
});
|
||||
}
|
||||
|
||||
private async _transcribeMainThread(audioData: Float32Array, langHint: string | undefined): Promise<string> {
|
||||
const pipelineFn = this.transcriber as (
|
||||
input: Float32Array,
|
||||
options?: Record<string, unknown>,
|
||||
) => Promise<{ text: string }>;
|
||||
|
||||
const result = await pipelineFn(audioData, {
|
||||
task: 'transcribe',
|
||||
...(langHint ? { language: langHint } : {}),
|
||||
});
|
||||
|
||||
return (result?.text ?? '').trim();
|
||||
}
|
||||
|
||||
private async _decodeToFloat32(blob: Blob): Promise<Float32Array | null> {
|
||||
if (!this.audioContext) return null;
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
let audioBuffer: AudioBuffer;
|
||||
try {
|
||||
audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const origRate = audioBuffer.sampleRate;
|
||||
const origData = audioBuffer.getChannelData(0);
|
||||
const targetRate = WHISPER_SAMPLE_RATE;
|
||||
|
||||
if (origRate === targetRate) {
|
||||
return new Float32Array(origData);
|
||||
}
|
||||
|
||||
const ratio = origRate / targetRate;
|
||||
const newLength = Math.ceil(origData.length / ratio);
|
||||
const result = new Float32Array(newLength);
|
||||
for (let i = 0; i < newLength; i++) {
|
||||
const origIdx = i * ratio;
|
||||
const idx0 = Math.floor(origIdx);
|
||||
const idx1 = Math.min(idx0 + 1, origData.length - 1);
|
||||
const frac = origIdx - idx0;
|
||||
result[i] = origData[idx0] * (1 - frac) + origData[idx1] * frac;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private _resolveLanguageHint(): string | undefined {
|
||||
if (this.lang && this.lang !== 'auto') {
|
||||
return this.lang.split('-')[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private _pickMimeType(): string {
|
||||
const candidates = [
|
||||
'audio/webm;codecs=opus',
|
||||
'audio/webm',
|
||||
'audio/ogg;codecs=opus',
|
||||
'audio/ogg',
|
||||
'audio/mp4',
|
||||
];
|
||||
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
|
||||
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
export const wasmSttService = new WasmSttService();
|
||||
@@ -1,92 +0,0 @@
|
||||
/**
|
||||
* Web Worker for off-main-thread Whisper transcription.
|
||||
*
|
||||
* Receives `{ type: 'load', modelId }` to load a model, then
|
||||
* `{ type: 'transcribe', audio: Float32Array (transferred buffer), language? }`
|
||||
* to run inference. Posts progress, results, and errors back.
|
||||
*/
|
||||
|
||||
import { pipeline, env } from '@xenova/transformers';
|
||||
|
||||
let transcriber: unknown = null;
|
||||
|
||||
self.onmessage = async (e: MessageEvent) => {
|
||||
const { type } = e.data as { type: string };
|
||||
|
||||
if (type === 'load') {
|
||||
const { modelId } = e.data as { modelId: string };
|
||||
try {
|
||||
env.backends.onnx.wasm.numThreads = 1;
|
||||
|
||||
const fileDoneBytes = new Map<string, number>();
|
||||
let totalDone = 0;
|
||||
let totalEstimate = 0;
|
||||
|
||||
transcriber = await pipeline('automatic-speech-recognition', modelId, {
|
||||
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
|
||||
if (info.status === 'progress' && info.file) {
|
||||
const prevDone = fileDoneBytes.get(info.file) ?? 0;
|
||||
const currentDone = info.loaded ?? 0;
|
||||
const delta = Math.max(0, currentDone - prevDone);
|
||||
fileDoneBytes.set(info.file, currentDone);
|
||||
totalDone += delta;
|
||||
|
||||
if (info.total && info.total > totalEstimate) {
|
||||
totalEstimate = info.total;
|
||||
}
|
||||
|
||||
const effectiveTotal = Math.max(totalEstimate, totalDone);
|
||||
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
|
||||
self.postMessage({ type: 'progress', progress: pct });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
self.postMessage({ type: 'loaded' });
|
||||
} catch (err) {
|
||||
self.postMessage({
|
||||
type: 'error',
|
||||
error: err instanceof Error ? err.message : 'Failed to load model',
|
||||
});
|
||||
}
|
||||
} else if (type === 'transcribe') {
|
||||
if (!transcriber) {
|
||||
self.postMessage({ type: 'error', error: 'Model not loaded', seq: (e.data as { seq?: number }).seq });
|
||||
return;
|
||||
}
|
||||
|
||||
const { audio, language, seq } = e.data as { audio: ArrayBuffer; language?: string; seq?: number };
|
||||
|
||||
try {
|
||||
const samples = new Float32Array(audio);
|
||||
if (samples.length === 0) {
|
||||
self.postMessage({ type: 'error', error: 'Empty audio received', seq });
|
||||
return;
|
||||
}
|
||||
|
||||
self.postMessage({ type: 'log', text: `Transcribing ${samples.length} samples (${(samples.length / 16000).toFixed(1)}s)` });
|
||||
|
||||
const pipelineFn = transcriber as (
|
||||
input: Float32Array,
|
||||
options?: Record<string, unknown>,
|
||||
) => Promise<{ text: string }>;
|
||||
|
||||
const result = await pipelineFn(samples, {
|
||||
task: 'transcribe',
|
||||
...(language ? { language } : {}),
|
||||
});
|
||||
|
||||
self.postMessage({
|
||||
type: 'result',
|
||||
transcript: (result?.text ?? '').trim(),
|
||||
seq,
|
||||
});
|
||||
} catch (err) {
|
||||
self.postMessage({
|
||||
type: 'error',
|
||||
error: err instanceof Error ? err.message : 'Transcription failed',
|
||||
seq,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -23,10 +23,6 @@ import { getSyncConfig, subscribeToSyncConfigChanges } from "@/sync/sync-refs";
|
||||
|
||||
const MODELS_DEV_API_URL = "https://models.dev/api.json";
|
||||
const MODELS_DEV_PROXY_URL = "/api/openchamber/models-metadata";
|
||||
const STT_SILENCE_THRESHOLD_DB_MIN = -100;
|
||||
const STT_SILENCE_THRESHOLD_DB_MAX = 0;
|
||||
const STT_SILENCE_HOLD_MS_MIN = 250;
|
||||
const STT_SILENCE_HOLD_MS_MAX = 10000;
|
||||
|
||||
const FALLBACK_PROVIDER_ID = "opencode";
|
||||
const FALLBACK_MODEL_ID = "big-pickle";
|
||||
@@ -38,18 +34,19 @@ const GIT_UTILITY_PROVIDER_ID = "zen";
|
||||
const GIT_UTILITY_PREFERRED_MODEL_ID = "big-pickle";
|
||||
const PROVIDER_CONFIG_REFRESH_CONCURRENCY = 4;
|
||||
|
||||
const normalizeSttSilenceThresholdDb = (value: unknown): number | undefined => {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||
return undefined;
|
||||
const normalizeSttProvider = (value: unknown): 'local' | 'openai-compatible' | undefined => {
|
||||
if (value === 'local' || value === 'openai-compatible') {
|
||||
return value;
|
||||
}
|
||||
return Math.max(STT_SILENCE_THRESHOLD_DB_MIN, Math.min(STT_SILENCE_THRESHOLD_DB_MAX, value));
|
||||
};
|
||||
|
||||
const normalizeSttSilenceHoldMs = (value: unknown): number | undefined => {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||
return undefined;
|
||||
// Legacy providers: 'server' used an OpenAI-compatible endpoint;
|
||||
// 'browser' and 'wasm' map to the local default.
|
||||
if (value === 'server') {
|
||||
return 'openai-compatible';
|
||||
}
|
||||
return Math.max(STT_SILENCE_HOLD_MS_MIN, Math.min(STT_SILENCE_HOLD_MS_MAX, Math.round(value)));
|
||||
if (value === 'browser' || value === 'wasm') {
|
||||
return 'local';
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
interface OpenChamberDefaults {
|
||||
@@ -61,13 +58,11 @@ interface OpenChamberDefaults {
|
||||
defaultFileViewerPreview?: boolean;
|
||||
zenModel?: string;
|
||||
messageStreamTransport?: 'auto' | 'ws' | 'sse';
|
||||
sttProvider?: 'browser' | 'server' | 'wasm';
|
||||
sttProvider?: 'local' | 'openai-compatible';
|
||||
sttServerUrl?: string;
|
||||
wasmSttModel?: string;
|
||||
sttModel?: string;
|
||||
sttLocalModel?: string;
|
||||
sttLanguage?: string;
|
||||
sttSilenceThresholdDb?: number;
|
||||
sttSilenceHoldMs?: number;
|
||||
}
|
||||
|
||||
const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
||||
@@ -101,12 +96,11 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
||||
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
|
||||
? data.messageStreamTransport
|
||||
: undefined;
|
||||
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' || data?.sttProvider === 'wasm' ? data.sttProvider : undefined;
|
||||
const sttProvider = normalizeSttProvider(data?.sttProvider);
|
||||
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
|
||||
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
|
||||
const sttLocalModel = typeof data?.sttLocalModel === 'string' ? data.sttLocalModel.trim() : undefined;
|
||||
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
|
||||
const sttSilenceThresholdDb = normalizeSttSilenceThresholdDb(data?.sttSilenceThresholdDb);
|
||||
const sttSilenceHoldMs = normalizeSttSilenceHoldMs(data?.sttSilenceHoldMs);
|
||||
|
||||
return finish('runtime-settings', {
|
||||
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
|
||||
@@ -120,9 +114,8 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
||||
sttProvider,
|
||||
sttServerUrl,
|
||||
sttModel,
|
||||
sttLocalModel,
|
||||
sttLanguage,
|
||||
sttSilenceThresholdDb,
|
||||
sttSilenceHoldMs,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
@@ -149,12 +142,11 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
||||
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
|
||||
? data.messageStreamTransport
|
||||
: undefined;
|
||||
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' ? data.sttProvider : undefined;
|
||||
const sttProvider = normalizeSttProvider(data?.sttProvider);
|
||||
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
|
||||
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
|
||||
const sttLocalModel = typeof data?.sttLocalModel === 'string' ? data.sttLocalModel.trim() : undefined;
|
||||
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
|
||||
const sttSilenceThresholdDb = normalizeSttSilenceThresholdDb(data?.sttSilenceThresholdDb);
|
||||
const sttSilenceHoldMs = normalizeSttSilenceHoldMs(data?.sttSilenceHoldMs);
|
||||
|
||||
return finish('settings-route', {
|
||||
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
|
||||
@@ -168,9 +160,8 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
||||
sttProvider,
|
||||
sttServerUrl,
|
||||
sttModel,
|
||||
sttLocalModel,
|
||||
sttLanguage,
|
||||
sttSilenceThresholdDb,
|
||||
sttSilenceHoldMs,
|
||||
});
|
||||
} catch (error) {
|
||||
markStartupTrace('config.defaults:error', { error: error instanceof Error ? error.message : String(error) });
|
||||
@@ -989,33 +980,31 @@ interface ConfigStore {
|
||||
settingsZenModel: string | undefined;
|
||||
settingsMessageStreamTransport: 'auto' | 'ws' | 'sse';
|
||||
// Voice provider preference ('browser', 'openai', 'openai-compatible', or 'say' for macOS)
|
||||
voiceProvider: 'browser' | 'openai' | 'openai-compatible' | 'say';
|
||||
setVoiceProvider: (provider: 'browser' | 'openai' | 'openai-compatible' | 'say') => void;
|
||||
voiceProvider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say';
|
||||
setVoiceProvider: (provider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say') => void;
|
||||
// TTS settings
|
||||
speechRate: number;
|
||||
speechPitch: number;
|
||||
speechVolume: number;
|
||||
sayVoice: string;
|
||||
browserVoice: string;
|
||||
localTtsVoiceId: number;
|
||||
openaiVoice: string;
|
||||
openaiApiKey: string;
|
||||
openaiCompatibleUrl: string;
|
||||
openaiCompatibleApiKey: string;
|
||||
openaiCompatibleVoice: string;
|
||||
openaiCompatibleTtsModel: string;
|
||||
// STT (speech-to-text) settings
|
||||
sttProvider: 'browser' | 'server' | 'wasm';
|
||||
// STT (dictation) settings
|
||||
dictationEnabled: boolean;
|
||||
sttProvider: 'local' | 'openai-compatible';
|
||||
sttServerUrl: string;
|
||||
sttApiKey: string;
|
||||
sttModel: string;
|
||||
wasmSttModel: string;
|
||||
sttLocalModel: string;
|
||||
sttLanguage: string;
|
||||
sttSilenceThresholdDb: number;
|
||||
sttSilenceHoldMs: number;
|
||||
sttTranscribeOnStop: boolean;
|
||||
showMessageTTSButtons: boolean;
|
||||
ttsInputMode: 'sanitized' | 'raw';
|
||||
voiceModeEnabled: boolean;
|
||||
// Summarization settings
|
||||
summarizeMessageTTS: boolean;
|
||||
summarizeVoiceConversation: boolean;
|
||||
@@ -1026,24 +1015,22 @@ interface ConfigStore {
|
||||
setSpeechVolume: (volume: number) => void;
|
||||
setSayVoice: (voice: string) => void;
|
||||
setBrowserVoice: (voice: string) => void;
|
||||
setLocalTtsVoiceId: (voiceId: number) => void;
|
||||
setOpenaiVoice: (voice: string) => void;
|
||||
setOpenaiApiKey: (apiKey: string) => void;
|
||||
setOpenaiCompatibleUrl: (url: string) => void;
|
||||
setOpenaiCompatibleApiKey: (apiKey: string) => void;
|
||||
setOpenaiCompatibleVoice: (voice: string) => void;
|
||||
setOpenaiCompatibleTtsModel: (model: string) => void;
|
||||
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => void;
|
||||
setDictationEnabled: (enabled: boolean) => void;
|
||||
setSttProvider: (provider: 'local' | 'openai-compatible') => void;
|
||||
setSttServerUrl: (url: string) => void;
|
||||
setSttApiKey: (apiKey: string) => void;
|
||||
setSttModel: (model: string) => void;
|
||||
setWasmSttModel: (model: string) => void;
|
||||
setSttLocalModel: (model: string) => void;
|
||||
setSttLanguage: (lang: string) => void;
|
||||
setSttSilenceThresholdDb: (db: number) => void;
|
||||
setSttSilenceHoldMs: (ms: number) => void;
|
||||
setSttTranscribeOnStop: (enabled: boolean) => void;
|
||||
setShowMessageTTSButtons: (show: boolean) => void;
|
||||
setTtsInputMode: (mode: 'sanitized' | 'raw') => void;
|
||||
setVoiceModeEnabled: (enabled: boolean) => void;
|
||||
setSummarizeMessageTTS: (enabled: boolean) => void;
|
||||
setSummarizeVoiceConversation: (enabled: boolean) => void;
|
||||
setSummarizeCharacterThreshold: (threshold: number) => void;
|
||||
@@ -1136,7 +1123,7 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
voiceProvider: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('voiceProvider');
|
||||
if (saved === 'openai' || saved === 'browser' || saved === 'say' || saved === 'openai-compatible') return saved;
|
||||
if (saved === 'openai' || saved === 'browser' || saved === 'local' || saved === 'say' || saved === 'openai-compatible') return saved;
|
||||
}
|
||||
return 'browser';
|
||||
})(),
|
||||
@@ -1179,6 +1166,17 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
return 'Samantha';
|
||||
})(),
|
||||
// Local (Kokoro) TTS speaker id - load from localStorage or default to 0
|
||||
localTtsVoiceId: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('localTtsVoiceId');
|
||||
if (saved !== null) {
|
||||
const parsed = Number.parseInt(saved, 10);
|
||||
if (Number.isInteger(parsed) && parsed >= 0) return parsed;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
})(),
|
||||
// Browser voice - load from localStorage or default to empty (auto-select)
|
||||
browserVoice: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -1235,17 +1233,24 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
return 'kokoro';
|
||||
})(),
|
||||
// STT provider: 'browser' (Web Speech API), 'server' (OpenAI-compat), 'wasm' (local Whisper)
|
||||
// Voice input (dictation) master toggle - default enabled
|
||||
dictationEnabled: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('dictationEnabled');
|
||||
if (saved === 'false') return false;
|
||||
}
|
||||
return true;
|
||||
})(),
|
||||
// STT provider: 'local' (server-side sherpa-onnx) or 'openai-compatible'
|
||||
sttProvider: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttProvider');
|
||||
if (saved === 'browser' || saved === 'server' || saved === 'wasm') return saved;
|
||||
// Electron/Chromium's Web Speech API requires Google API keys
|
||||
// not available in Electron, so default to WASM local Whisper.
|
||||
const electron = (window as unknown as { __OPENCHAMBER_ELECTRON__?: { runtime?: string } }).__OPENCHAMBER_ELECTRON__;
|
||||
if (electron?.runtime === 'electron') return 'wasm' as const;
|
||||
if (saved === 'local' || saved === 'openai-compatible') return saved;
|
||||
// Migrate legacy providers: 'server' used an OpenAI-compatible
|
||||
// endpoint; 'browser' and 'wasm' map to the local default.
|
||||
if (saved === 'server') return 'openai-compatible' as const;
|
||||
}
|
||||
return 'browser' as const;
|
||||
return 'local' as const;
|
||||
})(),
|
||||
sttServerUrl: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -1268,12 +1273,12 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
return 'deepdml/faster-whisper-large-v3-turbo-ct2';
|
||||
})(),
|
||||
wasmSttModel: (() => {
|
||||
sttLocalModel: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('wasmSttModel');
|
||||
const saved = localStorage.getItem('sttLocalModel');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return 'Xenova/whisper-base.en';
|
||||
return 'parakeet-tdt-0.6b-v2-int8';
|
||||
})(),
|
||||
sttLanguage: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -1282,33 +1287,6 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
sttSilenceThresholdDb: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttSilenceThresholdDb');
|
||||
if (saved) {
|
||||
const parsed = parseFloat(saved);
|
||||
if (!isNaN(parsed)) return parsed;
|
||||
}
|
||||
}
|
||||
return -45;
|
||||
})(),
|
||||
sttSilenceHoldMs: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttSilenceHoldMs');
|
||||
if (saved) {
|
||||
const parsed = parseInt(saved, 10);
|
||||
if (!isNaN(parsed)) return parsed;
|
||||
}
|
||||
}
|
||||
return 1500;
|
||||
})(),
|
||||
sttTranscribeOnStop: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sttTranscribeOnStop');
|
||||
if (saved === 'true') return true;
|
||||
}
|
||||
return false;
|
||||
})(),
|
||||
// Show TTS buttons on messages - disabled by default until user enables it
|
||||
showMessageTTSButtons: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -1324,14 +1302,6 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
return 'sanitized' as const;
|
||||
})(),
|
||||
// Voice mode enabled - load from localStorage or default to false
|
||||
voiceModeEnabled: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('voiceModeEnabled');
|
||||
if (saved === 'true') return true;
|
||||
}
|
||||
return false;
|
||||
})(),
|
||||
// Summarization settings
|
||||
summarizeMessageTTS: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -2060,9 +2030,8 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
sttProvider: openChamberDefaults.sttProvider ?? state.sttProvider,
|
||||
sttServerUrl: openChamberDefaults.sttServerUrl ?? state.sttServerUrl,
|
||||
sttModel: openChamberDefaults.sttModel ?? state.sttModel,
|
||||
sttLocalModel: openChamberDefaults.sttLocalModel ?? state.sttLocalModel,
|
||||
sttLanguage: openChamberDefaults.sttLanguage ?? state.sttLanguage,
|
||||
sttSilenceThresholdDb: openChamberDefaults.sttSilenceThresholdDb ?? state.sttSilenceThresholdDb,
|
||||
sttSilenceHoldMs: openChamberDefaults.sttSilenceHoldMs ?? state.sttSilenceHoldMs,
|
||||
directoryScoped: {
|
||||
...state.directoryScoped,
|
||||
[directoryKey]: nextSnapshot,
|
||||
@@ -2800,7 +2769,7 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
});
|
||||
},
|
||||
|
||||
setVoiceProvider: (provider: 'browser' | 'openai' | 'openai-compatible' | 'say') => {
|
||||
setVoiceProvider: (provider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say') => {
|
||||
set({ voiceProvider: provider });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('voiceProvider', provider);
|
||||
@@ -2838,6 +2807,13 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
},
|
||||
|
||||
setLocalTtsVoiceId: (voiceId: number) => {
|
||||
set({ localTtsVoiceId: voiceId });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('localTtsVoiceId', String(voiceId));
|
||||
}
|
||||
},
|
||||
|
||||
setBrowserVoice: (voice: string) => {
|
||||
set({ browserVoice: voice });
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -2887,7 +2863,15 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
},
|
||||
|
||||
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => {
|
||||
setDictationEnabled: (enabled: boolean) => {
|
||||
set({ dictationEnabled: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('dictationEnabled', String(enabled));
|
||||
}
|
||||
updateDesktopSettings({ dictationEnabled: enabled }).catch(() => {});
|
||||
},
|
||||
|
||||
setSttProvider: (provider: 'local' | 'openai-compatible') => {
|
||||
set({ sttProvider: provider });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttProvider', provider);
|
||||
@@ -2918,12 +2902,12 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
updateDesktopSettings({ sttModel: model }).catch(() => {});
|
||||
},
|
||||
|
||||
setWasmSttModel: (model: string) => {
|
||||
set({ wasmSttModel: model });
|
||||
setSttLocalModel: (model: string) => {
|
||||
set({ sttLocalModel: model });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('wasmSttModel', model);
|
||||
localStorage.setItem('sttLocalModel', model);
|
||||
}
|
||||
updateDesktopSettings({ wasmSttModel: model }).catch(() => {});
|
||||
updateDesktopSettings({ sttLocalModel: model }).catch(() => {});
|
||||
},
|
||||
|
||||
setSttLanguage: (lang: string) => {
|
||||
@@ -2934,30 +2918,6 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
updateDesktopSettings({ sttLanguage: lang }).catch(() => {});
|
||||
},
|
||||
|
||||
setSttSilenceThresholdDb: (db: number) => {
|
||||
set({ sttSilenceThresholdDb: db });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttSilenceThresholdDb', String(db));
|
||||
}
|
||||
updateDesktopSettings({ sttSilenceThresholdDb: db }).catch(() => {});
|
||||
},
|
||||
|
||||
setSttSilenceHoldMs: (ms: number) => {
|
||||
set({ sttSilenceHoldMs: ms });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttSilenceHoldMs', String(ms));
|
||||
}
|
||||
updateDesktopSettings({ sttSilenceHoldMs: ms }).catch(() => {});
|
||||
},
|
||||
|
||||
setSttTranscribeOnStop: (enabled: boolean) => {
|
||||
set({ sttTranscribeOnStop: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sttTranscribeOnStop', String(enabled));
|
||||
}
|
||||
updateDesktopSettings({ sttTranscribeOnStop: enabled }).catch(() => {});
|
||||
},
|
||||
|
||||
setShowMessageTTSButtons: (show: boolean) => {
|
||||
set({ showMessageTTSButtons: show });
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -2972,13 +2932,6 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
}
|
||||
},
|
||||
|
||||
setVoiceModeEnabled: (enabled: boolean) => {
|
||||
set({ voiceModeEnabled: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('voiceModeEnabled', String(enabled));
|
||||
}
|
||||
},
|
||||
|
||||
setSummarizeMessageTTS: (enabled: boolean) => {
|
||||
set({ summarizeMessageTTS: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
|
||||
@@ -190,7 +190,6 @@ function notifyMessageSent(sessionId: string): void {
|
||||
|
||||
export type { SyntheticContextPart } from "./input-store"
|
||||
export type { SessionMemoryState } from "./viewport-store"
|
||||
export type { VoiceStatus, VoiceMode } from "./voice-store"
|
||||
|
||||
export type NewSessionDraftState = {
|
||||
open: boolean
|
||||
|
||||
+2532
-2532
File diff suppressed because it is too large
Load Diff
@@ -1,7 +0,0 @@
|
||||
/**
|
||||
* Voice Store — voice connection and activity state.
|
||||
* Extracted from session-ui-store for subscription isolation.
|
||||
*/
|
||||
|
||||
export type VoiceStatus = "disconnected" | "connecting" | "connected" | "error"
|
||||
export type VoiceMode = "idle" | "speaking" | "listening"
|
||||
Reference in New Issue
Block a user