feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)

Complete rebuild of voice input on a server-authoritative streaming
architecture, replacing the legacy Web Speech / whole-blob / WASM engines
and the dead voice-agent layer (~4k lines removed).

Speech-to-text (dictation):
- Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with
  seq/ack ordering; buffered audio is retained and replayed on reconnect
- Server transcribes and streams live partial transcripts back;
  segments auto-commit every ~15s with silence suppression and adaptive
  finalization timeouts
- Local provider (default, zero config): sherpa-onnx models in a forked
  worker process — auto-download with progress, staged extraction with
  verification, corrupt-model auto-recovery, idle shutdown after 5 min
- Model catalog with settings picker (accuracy/speed ratings, sizes,
  download/delete): Parakeet TDT v2 (English) and v3 (25 European
  languages, auto-detected), Whisper base and tiny (multilingual, light)
- OpenAI-compatible provider for any Whisper endpoint
- Composer overlay with live transcript, volume meter, timer, and
  cancel / insert / insert-and-send actions; failed transcriptions keep
  their audio for retry or accepting the partial text as-is
- Configurable keyboard shortcut (default mod+alt+v) toggles dictation;
  Enter confirms and Escape cancels while recording
- Overlay is pixel-aligned with the composer (measured footer height,
  matching paddings/typography/gaps) — no layout shift when toggling

Text-to-speech:
- Local Kokoro provider (English, 11 voices) synthesized in the same
  worker via /api/dictation/tts/speak, managed by the shared model
  pipeline; sentence-pipelined playback keeps time-to-first-audio at
  ~1 sentence regardless of message length, and stop cancels in-flight
  synthesis
- Sanitizer keeps inline-code content (strips backticks only), reads
  interword slashes aloud, and removes only absolute file paths

Settings:
- Voice page unified: a single read-aloud toggle owns all playback
  options (the confusing "Enable Voice Mode" is gone); a new "Enable
  voice input" toggle (default on, persisted to settings.json) hides
  the composer mic entirely when disabled

Mobile and transport:
- iOS/Android microphone permissions added (dictation was previously
  impossible on mobile)
- Fixed Android WebSocket upgrades: the Capacitor WebView origin
  (https://localhost) was missing from the packaged-client allowlist,
  403-ing every WS connection — root cause of the old mobile SSE lock,
  which is now removed for all transports

Security and conventions:
- All HTTP routes sit behind the global /api auth gate; the WS upgrade
  explicitly validates the UI session and origin, with oc_url_token
  narrowly allowlisted and covered by tests; the dictation socket mints
  a fresh URL token before connecting
- Routes register before the generic OpenCode proxy; the client goes
  through runtimeFetch/getRuntimeUrlResolver, and runtime switches
  reset the dictation socket
- VS Code deliberately reports dictation as unavailable (no server
  process in that runtime)

CI: workflow Node bumped 20 -> 22 to match the repo engines and fix
better-sqlite3 installs broken by node-gyp@latest on Node 20.

New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64,
Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
This commit is contained in:
Bohdan Triapitsyn
2026-07-04 02:48:07 +03:00
committed by GitHub
parent 3f5151d424
commit de1b85ac56
89 changed files with 8740 additions and 6061 deletions
+2 -5
View File
@@ -44,7 +44,6 @@ import { ConfigUpdateOverlay } from '@/components/ui/ConfigUpdateOverlay';
import { AboutDialog } from '@/components/ui/AboutDialog';
import { RuntimeAPIProvider } from '@/contexts/RuntimeAPIProvider';
import { registerRuntimeAPIs } from '@/contexts/runtimeAPIRegistry';
import { VoiceProvider } from '@/components/voice';
import { useUIStore } from '@/stores/useUIStore';
import { useGitHubAuthStore } from '@/stores/useGitHubAuthStore';
import { useFeatureFlagsStore } from '@/stores/useFeatureFlagsStore';
@@ -928,8 +927,8 @@ function App({ apis }: AppProps) {
}
// Always mount the full provider tree to avoid remounts when isInitialized
// flips from false → true. FireworksProvider and VoiceProvider are lightweight
// shells; their heavy children are only activated when actually needed.
// flips from false → true. FireworksProvider is a lightweight shell; its
// heavy children are only activated when actually needed.
const isBootShell = !isInitialized && !isDesktopRuntime;
return (
@@ -937,7 +936,6 @@ function App({ apis }: AppProps) {
<SyncProvider key={runtimeEndpointEpoch} sdk={opencodeClient.getSdkClient()} directory={currentDirectory || ''}>
<RuntimeAPIProvider apis={apis}>
<FireworksProvider>
<VoiceProvider>
<TooltipProvider delayDuration={300} skipDelayDuration={150}>
<div className={isDesktopRuntime ? 'h-full text-foreground bg-transparent' : 'h-full text-foreground bg-background'}>
<SyncAppEffects embeddedBackgroundWorkEnabled={embeddedBackgroundWorkEnabled} />
@@ -955,7 +953,6 @@ function App({ apis }: AppProps) {
)}
</div>
</TooltipProvider>
</VoiceProvider>
</FireworksProvider>
</RuntimeAPIProvider>
</SyncProvider>
+53 -4
View File
@@ -1,6 +1,6 @@
import React from 'react';
import { Textarea } from '@/components/ui/textarea';
import { BrowserVoiceButton } from '@/components/voice';
import { ComposerDictation } from '@/components/dictation/ComposerDictation';
// sessionStore removed — currentSessionId comes from useSessionUIStore
import { useConfigStore } from '@/stores/useConfigStore';
import { useUIStore } from '@/stores/useUIStore';
@@ -381,7 +381,7 @@ const getProjectIconColor = (projectColor?: string | null): string | undefined =
};
const MemoModelControls = React.memo(ModelControls);
const MemoBrowserVoiceButton = React.memo(BrowserVoiceButton);
const MemoComposerDictation = React.memo(ComposerDictation);
const MemoMobileAgentButton = React.memo(MobileAgentButton);
const MemoMobileModelButton = React.memo(MobileModelButton);
const MemoStatusRow = React.memo(StatusRow);
@@ -2318,6 +2318,33 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
void handleSubmitRef.current();
}, []);
// Dictation: insert the transcript inline; optionally submit immediately.
// getCurrentInputSnapshot reads textareaRef.current.value first, so setting
// it synchronously lets handleSubmit pick up the text in the same tick.
const handleDictationInsert = React.useCallback((text: string) => {
setMessage((prev) => {
const next = appendInlineText(prev, text);
const textarea = textareaRef.current;
if (textarea) {
textarea.value = next;
}
return next;
});
setTimeout(() => {
textareaRef.current?.focus();
}, 0);
}, []);
const handleDictationInsertAndSend = React.useCallback((text: string) => {
const textarea = textareaRef.current;
const next = appendInlineText(textarea?.value ?? messageRef.current, text);
if (textarea) {
textarea.value = next;
}
setMessage(next);
void handleSubmitRef.current();
}, []);
// Preset chips rendered outside this component (e.g. under the welcome
// message on narrow surfaces) request a submit via the input store; consume
// it here so it routes through the same command-aware submit path.
@@ -4420,6 +4447,9 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
: undefined}
/>
)}
{/* Positioning context for the dictation overlay: covers the
text area + footer exactly, excluding MobileSessionStatusBar. */}
<div className={cn('relative flex flex-col', isDesktopExpanded && 'flex-1 min-h-0')}>
<div className={cn("overflow-hidden", isDesktopExpanded && 'flex flex-1 min-h-0 flex-col')}>
<div className="flex items-center gap-1 px-3 pt-1 flex-wrap relative z-10">
<AttachedVSCodeFileChips onShowPopup={handleShowAttachmentPreview} />
@@ -4559,7 +4589,16 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
/>
</div>
<div className="flex items-center gap-x-1 flex-shrink-0">
<MemoBrowserVoiceButton />
<MemoComposerDictation
radius={chatInputRadius}
isMobile={isMobile}
footerIconButtonClass={footerIconButtonClass}
footerPaddingClass={footerPaddingClass}
iconSizeClass={iconSizeClass}
sendIconSizeClass={sendIconSizeClass}
onInsert={handleDictationInsert}
onInsertAndSend={handleDictationInsertAndSend}
/>
<ComposerActionButtons
isMobile={isMobile}
footerIconButtonClass={footerIconButtonClass}
@@ -4614,7 +4653,16 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
</div>
<div className={cn('flex items-center flex-1 justify-end', footerGapClass, 'md:gap-x-3')}>
<MemoModelControls className={cn('flex-1 min-w-0 justify-end')} />
<MemoBrowserVoiceButton />
<MemoComposerDictation
radius={chatInputRadius}
isMobile={isMobile}
footerIconButtonClass={footerIconButtonClass}
footerPaddingClass={footerPaddingClass}
iconSizeClass={iconSizeClass}
sendIconSizeClass={sendIconSizeClass}
onInsert={handleDictationInsert}
onInsertAndSend={handleDictationInsertAndSend}
/>
<ComposerActionButtons
isMobile={isMobile}
footerIconButtonClass={footerIconButtonClass}
@@ -4633,6 +4681,7 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
</>
)}
</div>
</div>
{/* Mobile session panel: slide-up overlay toggled by MobileSessionPanelTrigger. */}
{isMobile && <MobileSessionStatusBar />}
@@ -0,0 +1,439 @@
/**
* Composer dictation controls: a mic button for the composer footer plus a
* full-composer overlay while dictation is active (recording, transcribing,
* or failed). The overlay mirrors the composer's own layout — the transcript
* area uses the same paddings/typography as the textarea and the action row
* reuses the footer icon-button styling — so toggling dictation causes no
* vertical shift.
*/
import React from 'react';
import { Icon } from '@/components/icon/Icon';
import { useI18n } from '@/lib/i18n';
import { useThemeSystem } from '@/contexts/useThemeSystem';
import { cn } from '@/lib/utils';
import { runtimeFetch } from '@/lib/runtime-fetch';
import { useDictation } from '@/hooks/useDictation';
import { isDictationCaptureSupported } from '@/lib/dictation/use-dictation-audio-source';
import { isVSCodeRuntime } from '@/lib/desktop';
import { useConfigStore } from '@/stores/useConfigStore';
import { useUIStore } from '@/stores/useUIStore';
import { formatShortcutForDisplay, getEffectiveShortcutCombo } from '@/lib/shortcuts';
interface ComposerDictationProps {
radius?: number | string;
isMobile: boolean;
footerIconButtonClass: string;
footerPaddingClass: string;
iconSizeClass: string;
sendIconSizeClass: string;
disabled?: boolean;
onInsert: (text: string) => void;
onInsertAndSend: (text: string) => void;
}
const formatDuration = (seconds: number): string => {
const mins = Math.floor(seconds / 60);
const secs = seconds % 60;
return `${mins}:${String(secs).padStart(2, '0')}`;
};
const VolumeMeter: React.FC<{ volume: number }> = ({ volume }) => {
const { currentTheme } = useThemeSystem();
return (
<div
className="h-1.5 w-16 flex-shrink-0 overflow-hidden rounded-full"
style={{ backgroundColor: currentTheme.colors.interactive.border }}
aria-hidden="true"
>
<div
className="h-full rounded-full transition-[width] duration-75"
style={{
width: `${Math.round(Math.min(1, volume) * 100)}%`,
backgroundColor: currentTheme.colors.primary.base,
}}
/>
</div>
);
};
/**
* Polls the dictation status route while the local model is downloading and
* returns the download percent (null while unknown / not downloading).
*/
const useModelDownloadProgress = (active: boolean): number | null => {
const sttLocalModel = useConfigStore((state) => state.sttLocalModel);
const [percent, setPercent] = React.useState<number | null>(null);
React.useEffect(() => {
if (!active) {
setPercent(null);
return;
}
let cancelled = false;
const poll = async () => {
try {
const response = await runtimeFetch('/api/dictation/status', {
query: { provider: 'local', localModel: sttLocalModel },
});
if (!response.ok || cancelled) {
return;
}
const data = await response.json();
const model = Array.isArray(data?.models)
? data.models.find((m: { id: string }) => m.id === sttLocalModel)
: null;
if (!cancelled) {
setPercent(typeof model?.downloadProgress === 'number' ? model.downloadProgress : null);
}
} catch {
// Display-only; keep the previous value.
}
};
void poll();
const interval = setInterval(() => {
void poll();
}, 2000);
return () => {
cancelled = true;
clearInterval(interval);
};
}, [active, sttLocalModel]);
return active ? percent : null;
};
export const ComposerDictation: React.FC<ComposerDictationProps> = ({
radius,
isMobile,
footerIconButtonClass,
footerPaddingClass,
iconSizeClass,
sendIconSizeClass,
disabled,
onInsert,
onInsertAndSend,
}) => {
const { t } = useI18n();
const { currentTheme } = useThemeSystem();
const dictationEnabled = useConfigStore((state) => state.dictationEnabled);
const shortcutOverrides = useUIStore((state) => state.shortcutOverrides);
const dictationShortcut = formatShortcutForDisplay(getEffectiveShortcutCombo('toggle_dictation', shortcutOverrides));
// The dictation server (WebSocket + STT worker) lives in the OpenChamber
// web server; the VS Code bridge has no server process for it.
const [supported] = React.useState(() => !isVSCodeRuntime() && isDictationCaptureSupported());
const pendingActionRef = React.useRef<'insert' | 'send' | null>(null);
const onInsertRef = React.useRef(onInsert);
const onInsertAndSendRef = React.useRef(onInsertAndSend);
React.useEffect(() => {
onInsertRef.current = onInsert;
onInsertAndSendRef.current = onInsertAndSend;
}, [onInsert, onInsertAndSend]);
const dictation = useDictation({
onTranscript: (text) => {
const action = pendingActionRef.current;
pendingActionRef.current = null;
if (action === 'send') {
onInsertAndSendRef.current(text);
} else {
onInsertRef.current(text);
}
},
});
const {
status,
partialTranscript,
volume,
duration,
error,
errorReason,
startDictation,
confirmDictation,
cancelDictation,
retryFailedDictation,
acceptPartialTranscript,
discardFailedDictation,
} = dictation;
const isModelDownloading = status === 'recording' && errorReason === 'model_download_in_progress';
const downloadPercent = useModelDownloadProgress(isModelDownloading);
const statusRef = React.useRef(status);
React.useEffect(() => {
statusRef.current = status;
}, [status]);
// Keyboard shortcut (toggle_dictation): idle -> start recording,
// recording -> confirm and insert. Dispatched by useKeyboardShortcuts.
React.useEffect(() => {
const onToggle = () => {
if (statusRef.current === 'idle') {
void startDictation();
} else if (statusRef.current === 'recording') {
pendingActionRef.current = 'insert';
void confirmDictation();
}
};
window.addEventListener('openchamber:dictation-toggle', onToggle);
return () => window.removeEventListener('openchamber:dictation-toggle', onToggle);
}, [startDictation, confirmDictation]);
// While recording: Enter confirms (insert), Escape cancels. Capture-phase
// so the composer's own Enter-to-send never fires underneath the overlay.
React.useEffect(() => {
if (status !== 'recording') {
return;
}
const onKeyDown = (event: KeyboardEvent) => {
if (event.isComposing) {
return;
}
if (event.key === 'Enter' && !event.shiftKey && !event.metaKey && !event.ctrlKey && !event.altKey) {
event.preventDefault();
event.stopPropagation();
pendingActionRef.current = 'insert';
void confirmDictation();
return;
}
if (event.key === 'Escape') {
event.preventDefault();
event.stopPropagation();
void cancelDictation();
}
};
window.addEventListener('keydown', onKeyDown, { capture: true });
return () => window.removeEventListener('keydown', onKeyDown, { capture: true });
}, [status, confirmDictation, cancelDictation]);
// Pixel-parity with the composer: the real footer row is taller than our
// buttons (its height comes from the tallest control, e.g. the model
// picker), so measure it — it stays mounted underneath the overlay — and
// give our action row the same height so the icons line up exactly.
const overlayRef = React.useRef<HTMLDivElement | null>(null);
const [footerHeight, setFooterHeight] = React.useState<number | null>(null);
const isActiveStatus = status !== 'idle';
React.useLayoutEffect(() => {
if (!isActiveStatus) {
return;
}
// The overlay is rendered inside the composer footer itself, so the
// real footer is an ancestor, not a sibling.
const realFooter = overlayRef.current?.closest<HTMLElement>('[data-chat-input-footer="true"]');
if (realFooter && realFooter.offsetHeight > 0) {
setFooterHeight(realFooter.offsetHeight);
}
}, [isActiveStatus]);
if (!supported || !dictationEnabled) {
return null;
}
const isActive = status !== 'idle';
const confirmWith = (action: 'insert' | 'send') => {
pendingActionRef.current = action;
void confirmDictation();
};
const retry = () => {
pendingActionRef.current = 'insert';
void retryFailedDictation();
};
const placeholderText = (() => {
if (status === 'failed') {
return '';
}
if (status === 'uploading') {
return t('chat.dictation.processing');
}
if (isModelDownloading) {
return downloadPercent !== null
? t('chat.dictation.downloadingModelProgress', { percent: String(downloadPercent) })
: t('chat.dictation.downloadingModel');
}
return t('chat.dictation.listening');
})();
return (
<>
<button
type="button"
className={footerIconButtonClass}
onClick={() => {
void startDictation();
}}
disabled={disabled || isActive}
title={dictationShortcut ? `${t('chat.dictation.start')} (${dictationShortcut})` : t('chat.dictation.start')}
aria-label={t('chat.dictation.start')}
>
<Icon name="mic" className={cn(iconSizeClass, 'text-current')} />
</button>
{isActive ? (
<div
ref={overlayRef}
// overflow-x/y split on purpose: mobile.css rewrites the
// shorthand `.overflow-hidden` to overflow-y:auto on touch
// devices, which painted a phantom scrollbar on Android.
className="absolute inset-0 z-50 flex flex-col overflow-x-hidden overflow-y-hidden"
style={{
borderRadius: radius,
// Must match the composer box background exactly so the
// overlay reads as the same surface, not a layer on top.
backgroundColor: currentTheme.colors.surface.subtle,
}}
role="dialog"
aria-label={t('chat.dictation.overlayAria')}
>
<div
className={cn(
// Text paddings match the composer textarea, plus the
// 4px (pt-1) attachment-chips row that always renders
// above it: desktop 16+4px, mobile 10+4px from the top.
// min-h-0 (not a fixed min height): the mobile composer
// is shorter than 52px of text area + footer, and a
// fixed min pushed the action row 4px below the real
// footer. The area must shrink to whatever space the
// underlying composer actually has.
'flex-1 min-h-0 overflow-y-auto px-3',
isMobile ? 'pt-3.5 pb-2.5' : 'pt-5 pb-2',
)}
>
{partialTranscript ? (
<p className="typography-markdown md:typography-ui-label whitespace-pre-wrap" style={{ color: currentTheme.colors.surface.foreground }}>
{partialTranscript}
</p>
) : (
<p className="typography-markdown md:typography-ui-label" style={{ color: currentTheme.colors.surface.mutedForeground }}>
{placeholderText}
</p>
)}
{status === 'failed' ? (
<p className="typography-meta mt-1" style={{ color: currentTheme.colors.status.error }}>
{error || t('chat.dictation.failed')}
</p>
) : null}
{status === 'recording' && error && !isModelDownloading ? (
<p className="typography-meta mt-1" style={{ color: currentTheme.colors.status.warning }}>
{error}
</p>
) : null}
</div>
<div
className={cn('flex flex-shrink-0 items-center gap-x-3', footerPaddingClass)}
style={footerHeight ? { height: footerHeight } : undefined}
>
{status === 'recording' ? (
<>
<span className="relative ml-1 flex h-2 w-2 flex-shrink-0" aria-hidden="true">
<span
className="absolute inline-flex h-full w-full animate-ping rounded-full opacity-60"
style={{ backgroundColor: currentTheme.colors.status.error }}
/>
<span
className="relative inline-flex h-2 w-2 rounded-full"
style={{ backgroundColor: currentTheme.colors.status.error }}
/>
</span>
<VolumeMeter volume={volume} />
<span className="typography-meta tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
{formatDuration(duration)}
</span>
</>
) : status === 'uploading' ? (
<Icon name="loader-4" className="ml-1 h-4 w-4 animate-spin" style={{ color: currentTheme.colors.surface.mutedForeground }} />
) : null}
{/* Same inter-control gap as the composer's right cluster:
gap-x-1 on mobile, md:gap-x-3 on desktop. */}
<div className={cn('ml-auto flex items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
{status === 'recording' ? (
<>
<button
type="button"
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
onClick={() => {
void cancelDictation();
}}
title={t('chat.dictation.cancel')}
aria-label={t('chat.dictation.cancel')}
>
<Icon name="close" className={iconSizeClass} />
</button>
<button
type="button"
className={footerIconButtonClass}
onClick={() => confirmWith('insert')}
title={t('chat.dictation.insert')}
aria-label={t('chat.dictation.insert')}
>
<Icon name="check" className={iconSizeClass} />
</button>
<button
type="button"
className={cn(footerIconButtonClass, 'text-primary hover:text-primary')}
onClick={() => confirmWith('send')}
title={t('chat.dictation.insertAndSend')}
aria-label={t('chat.dictation.insertAndSend')}
>
<Icon name="send-plane-2" className={sendIconSizeClass} />
</button>
</>
) : status === 'uploading' ? (
<button
type="button"
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
onClick={() => {
void cancelDictation();
}}
title={t('chat.dictation.cancel')}
aria-label={t('chat.dictation.cancel')}
>
<Icon name="close" className={iconSizeClass} />
</button>
) : (
<>
<button
type="button"
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
onClick={discardFailedDictation}
title={t('chat.dictation.discard')}
aria-label={t('chat.dictation.discard')}
>
<Icon name="close" className={iconSizeClass} />
</button>
<button
type="button"
className={footerIconButtonClass}
onClick={retry}
title={t('chat.dictation.retry')}
aria-label={t('chat.dictation.retry')}
>
<Icon name="refresh" className={iconSizeClass} />
</button>
{partialTranscript.trim() ? (
<button
type="button"
className={footerIconButtonClass}
onClick={() => {
pendingActionRef.current = 'insert';
acceptPartialTranscript();
}}
title={t('chat.dictation.insert')}
aria-label={t('chat.dictation.insert')}
>
<Icon name="check" className={iconSizeClass} />
</button>
) : null}
</>
)}
</div>
</div>
</div>
) : null}
</>
);
};
@@ -7,7 +7,6 @@ import type { ThemeMode } from '@/types/theme';
import { useUIStore } from '@/stores/useUIStore';
import { useMessageQueueStore, type FollowUpBehavior } from '@/stores/messageQueueStore';
import { cn } from '@/lib/utils';
import { isCapacitorApp } from '@/lib/platform';
import { Button } from '@/components/ui/button';
import { Checkbox } from '@/components/ui/checkbox';
import { NumberInput } from '@/components/ui/number-input';
@@ -322,10 +321,7 @@ export const OpenChamberVisualSettings: React.FC<OpenChamberVisualSettingsProps>
const setShowSplitAssistantMessageActions = useUIStore(state => state.setShowSplitAssistantMessageActions);
const messageStreamTransport = useConfigStore((state) => state.settingsMessageStreamTransport);
const setMessageStreamTransport = useConfigStore((state) => state.setSettingsMessageStreamTransport);
// Capacitor apps are locked to SSE (native WebSocket streaming is unreliable on mobile);
// sync-context forces it too. Show SSE selected and disable the other options here.
const isCapacitorAppRuntime = React.useMemo(() => isCapacitorApp(), []);
const effectiveMessageStreamTransport = isCapacitorAppRuntime ? 'sse' : messageStreamTransport;
const effectiveMessageStreamTransport = messageStreamTransport;
const settingsDefaultFileViewerPreview = useConfigStore((state) => state.settingsDefaultFileViewerPreview);
const setSettingsDefaultFileViewerPreview = useConfigStore((state) => state.setSettingsDefaultFileViewerPreview);
const isSettingsDialogOpen = useUIStore(state => state.isSettingsDialogOpen);
@@ -1531,7 +1527,6 @@ export const OpenChamberVisualSettings: React.FC<OpenChamberVisualSettingsProps>
variant="chip"
size="xs"
aria-pressed={effectiveMessageStreamTransport === option.id}
disabled={isCapacitorAppRuntime && option.id !== 'sse'}
className="!font-normal"
onClick={() => handleMessageStreamTransportChange(option.id)}
>
@@ -1,5 +1,4 @@
import React, { useEffect, useState, useCallback, useMemo } from 'react';
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
import { useConfigStore } from '@/stores/useConfigStore';
import { useDeviceInfo } from '@/lib/device';
@@ -11,103 +10,362 @@ import {
SelectValue,
} from '@/components/ui/select';
import { Checkbox } from '@/components/ui/checkbox';
import { Radio } from '@/components/ui/radio';
import { Button } from '@/components/ui/button';
import { NumberInput } from '@/components/ui/number-input';
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
import { Icon } from "@/components/icon/Icon";
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
import { audioStreamService } from '@/lib/voice/audioStreamService';
import { wasmSttService, WASM_MODELS } from '@/lib/voice/wasmSttService';
import type { WasmModelStatus } from '@/lib/voice/wasmSttService';
import { cn } from '@/lib/utils';
import { runtimeFetch } from '@/lib/runtime-fetch';
import { useI18n } from '@/lib/i18n';
import { useLocalTTS } from '@/hooks/useLocalTTS';
import { disposePreviewAudio } from './voicePreviewAudio';
const LANGUAGE_OPTIONS = [
{ value: 'en-US', label: 'English' },
{ value: 'es-ES', label: 'Español' },
{ value: 'fr-FR', label: 'Français' },
{ value: 'de-DE', label: 'Deutsch' },
{ value: 'ja-JP', label: '日本語' },
{ value: 'zh-CN', label: '中文' },
{ value: 'pt-BR', label: 'Português' },
{ value: 'it-IT', label: 'Italiano' },
{ value: 'ko-KR', label: '한국어' },
{ value: 'uk-UA', label: 'Українська' },
];
const WasmModelStatusIndicator = ({ modelId }: { modelId: string }) => {
const LOCAL_STT_MODELS = [
{
id: 'parakeet-tdt-0.6b-v2-int8',
labelKey: 'settings.voice.page.stt.model.parakeetV2',
badgeKey: 'settings.voice.page.stt.badge.bestForEnglish',
accuracy: 5,
speed: 5,
size: '460 MB',
},
{
id: 'parakeet-tdt-0.6b-v3-int8',
labelKey: 'settings.voice.page.stt.model.parakeetV3',
badgeKey: 'settings.voice.page.stt.badge.bestForMultilingual',
accuracy: 5,
speed: 4,
size: '465 MB',
},
{
id: 'whisper-base-int8',
labelKey: 'settings.voice.page.stt.model.whisperBase',
badgeKey: null,
accuracy: 3,
speed: 3,
size: '200 MB',
},
{
id: 'whisper-tiny-int8',
labelKey: 'settings.voice.page.stt.model.whisperTiny',
badgeKey: null,
accuracy: 2,
speed: 4,
size: '115 MB',
},
] as const;
interface DictationModelState {
id: string;
installed: boolean;
downloading: boolean;
downloadProgress: number | null;
downloadError: string | null;
}
const RatingBar = ({ value, label }: { value: number; label: string }) => (
<span className="flex items-center gap-1.5" title={`${label}: ${value}/5`}>
<span
className="h-1.5 w-12 flex-shrink-0 overflow-hidden rounded-full bg-[var(--interactive-border)]"
aria-hidden="true"
>
<span
className="block h-full rounded-full bg-[var(--primary-base)]"
style={{ width: `${(value / 5) * 100}%` }}
/>
</span>
<span className="typography-ui-compact text-muted-foreground">{label}</span>
</span>
);
const LocalModelPicker = ({
selectedModelId,
onSelect,
}: {
selectedModelId: string;
onSelect: (modelId: string) => void;
}) => {
const { t } = useI18n();
const [status, setStatus] = useState<WasmModelStatus>(wasmSttService.getModelStatus());
const [loading, setLoading] = useState(false);
const tUnsafe = useCallback((key: string) => t(key as Parameters<typeof t>[0]), [t]);
const [models, setModels] = useState<Map<string, DictationModelState>>(new Map());
const [requestingId, setRequestingId] = useState<string | null>(null);
useEffect(() => {
const handler = (s: WasmModelStatus) => setStatus(s);
wasmSttService.onModelStatusChange = handler;
return () => { wasmSttService.onModelStatusChange = null; };
const refresh = useCallback(async () => {
try {
const response = await runtimeFetch('/api/dictation/status', {
query: { provider: 'local' },
});
if (!response.ok) {
return;
}
const data = await response.json();
if (Array.isArray(data?.models)) {
setModels(new Map(data.models.map((m: DictationModelState) => [m.id, m])));
}
} catch {
// Display-only status; keep the previous state on fetch failure.
}
}, []);
const currentModelId = wasmSttService.getCurrentModelId();
const isLoadingOrDownloading = status.state === 'downloading' || status.state === 'loading';
// Reset local loading state when model finishes loading or errors
useEffect(() => {
if (status.state !== 'downloading' && status.state !== 'loading') {
setLoading(false);
}
}, [status.state]);
void refresh();
}, [refresh]);
const handleDownload = async () => {
setLoading(true);
const anyDownloading = Array.from(models.values()).some((m) => m.downloading);
useEffect(() => {
if (!anyDownloading) {
return;
}
const interval = setInterval(() => {
void refresh();
}, 2000);
return () => clearInterval(interval);
}, [anyDownloading, refresh]);
const handleDownload = async (modelId: string) => {
setRequestingId(modelId);
try {
await wasmSttService.loadModel(modelId);
await runtimeFetch(`/api/dictation/models/${encodeURIComponent(modelId)}/download`, {
method: 'POST',
});
await refresh();
} catch {
// Error is shown via status indicator
// Status refresh reports errors.
} finally {
setRequestingId(null);
}
};
if (status.state === 'ready' && currentModelId === modelId) {
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-green-600 dark:text-green-400">
{t('settings.voice.page.stt.wasmLoaded')}
</span>
</div>
);
}
const handleDelete = async (modelId: string) => {
setRequestingId(modelId);
try {
await runtimeFetch(`/api/dictation/models/${encodeURIComponent(modelId)}`, {
method: 'DELETE',
});
await refresh();
} catch {
// Status refresh reports errors.
} finally {
setRequestingId(null);
}
};
if (isLoadingOrDownloading) {
const progress = status.state === 'downloading' ? Math.round(status.progress) : undefined;
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-muted-foreground">
{status.state === 'downloading' ? t('settings.voice.page.stt.wasmDownloading') : t('settings.voice.page.stt.wasmLoading')}
{progress !== undefined ? ` (${progress}%)` : ''}
</span>
</div>
);
}
return (
<div role="radiogroup" aria-label={t('settings.voice.page.field.model')} className="mt-1 space-y-0">
{LOCAL_STT_MODELS.map((entry) => {
const selected = selectedModelId === entry.id;
const state = models.get(entry.id) ?? null;
return (
<div key={entry.id} className="flex w-full items-start gap-2 py-1.5">
<div className="pt-0.5">
<Radio
checked={selected}
onChange={() => onSelect(entry.id)}
ariaLabel={tUnsafe(entry.labelKey)}
/>
</div>
<div
className="min-w-0 flex-1 cursor-pointer"
role="button"
tabIndex={0}
onClick={() => onSelect(entry.id)}
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); onSelect(entry.id); } }}
>
<div className="flex flex-wrap items-center gap-x-2 gap-y-0.5">
<span className={cn('typography-ui-label font-normal', selected ? 'text-foreground' : 'text-foreground/50')}>
{tUnsafe(entry.labelKey)}
</span>
{entry.badgeKey ? (
<span
className="rounded px-1 text-[9px] font-medium uppercase leading-[14px] tracking-wide"
style={{
backgroundColor: 'var(--status-success-background)',
color: 'var(--status-success)',
border: '1px solid var(--status-success-border)',
}}
>
{tUnsafe(entry.badgeKey)}
</span>
) : null}
</div>
<div className="mt-0.5 flex flex-wrap items-center gap-x-4 gap-y-0.5">
<RatingBar value={entry.accuracy} label={t('settings.voice.page.stt.meta.accuracy')} />
<RatingBar value={entry.speed} label={t('settings.voice.page.stt.meta.speed')} />
<span className="typography-ui-compact tabular-nums text-muted-foreground">{entry.size}</span>
</div>
{state?.downloadError ? (
<p className="mt-0.5 typography-meta text-[var(--status-error)]">{state.downloadError}</p>
) : null}
</div>
<div className="flex w-14 flex-shrink-0 items-center justify-end gap-1 pt-0.5">
{state?.installed ? (
<>
<Button
variant="ghost"
size="xs"
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
disabled={requestingId === entry.id}
onClick={() => { void handleDelete(entry.id); }}
title={t('settings.voice.page.stt.modelDelete')}
aria-label={t('settings.voice.page.stt.modelDelete')}
>
<Icon name="delete-bin" className="h-4 w-4" />
</Button>
<Icon
name="checkbox-circle"
className="h-4 w-4 flex-shrink-0 text-[var(--status-success)]"
aria-label={t('settings.voice.page.stt.modelInstalled')}
/>
</>
) : state?.downloading ? (
<span className="flex items-center gap-1.5">
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
<span className="typography-ui-compact tabular-nums text-muted-foreground">
{typeof state.downloadProgress === 'number' ? `${state.downloadProgress}%` : ''}
</span>
</span>
) : (
<Button
variant="ghost"
size="xs"
className="h-6 w-6 p-0"
disabled={requestingId === entry.id}
onClick={() => { void handleDownload(entry.id); }}
title={t('settings.voice.page.stt.modelDownload')}
aria-label={t('settings.voice.page.stt.modelDownload')}
>
<Icon name="download" className="h-4 w-4" />
</Button>
)}
</div>
</div>
);
})}
</div>
);
};
if (status.state === 'error') {
// Partial download (cached progress): show retry button.
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-destructive">{status.error}</span>
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
{t('settings.voice.page.stt.wasmRetry')}
</Button>
</div>
);
/** Kokoro en_v0_19 speaker ids in sherpa-onnx order. */
const KOKORO_VOICE_OPTIONS = [
{ id: 0, label: 'Alloy (af)' },
{ id: 1, label: 'Bella (af)' },
{ id: 2, label: 'Nicole (af)' },
{ id: 3, label: 'Sarah (af)' },
{ id: 4, label: 'Sky (af)' },
{ id: 5, label: 'Adam (am)' },
{ id: 6, label: 'Michael (am)' },
{ id: 7, label: 'Emma (bf)' },
{ id: 8, label: 'Isabella (bf)' },
{ id: 9, label: 'George (bm)' },
{ id: 10, label: 'Lewis (bm)' },
];
const LOCAL_TTS_MODEL_ID = 'kokoro-en-v0_19';
const LocalTtsModelStatus = () => {
const { t } = useI18n();
const [model, setModel] = useState<DictationModelState | null>(null);
const [requesting, setRequesting] = useState(false);
const refresh = useCallback(async () => {
try {
const response = await runtimeFetch('/api/dictation/status', { query: { provider: 'local' } });
if (!response.ok) {
return;
}
const data = await response.json();
const entry = Array.isArray(data?.ttsModels)
? data.ttsModels.find((m: DictationModelState) => m.id === LOCAL_TTS_MODEL_ID)
: null;
if (entry) {
setModel(entry);
}
} catch {
// Display-only status; keep the previous state on fetch failure.
}
}, []);
useEffect(() => {
void refresh();
}, [refresh]);
useEffect(() => {
if (!model?.downloading) {
return;
}
const interval = setInterval(() => {
void refresh();
}, 2000);
return () => clearInterval(interval);
}, [model?.downloading, refresh]);
const request = async (method: 'POST' | 'DELETE') => {
setRequesting(true);
try {
const path = method === 'POST'
? `/api/dictation/models/${LOCAL_TTS_MODEL_ID}/download`
: `/api/dictation/models/${LOCAL_TTS_MODEL_ID}`;
await runtimeFetch(path, { method });
await refresh();
} catch {
// Status refresh reports errors.
} finally {
setRequesting(false);
}
};
if (!model) {
return null;
}
return (
<div className="flex items-center gap-2">
<span className="typography-ui-compact text-muted-foreground">
{t('settings.voice.page.stt.wasmNotLoaded')}
</span>
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
{t('settings.voice.page.stt.wasmDownload')}
</Button>
<div className="flex items-center gap-2 py-1.5">
<span className="typography-ui-label text-foreground">Kokoro</span>
<span className="typography-ui-compact tabular-nums text-muted-foreground">305 MB</span>
{model.installed ? (
<>
<Icon
name="checkbox-circle"
className="h-4 w-4 text-[var(--status-success)]"
aria-label={t('settings.voice.page.stt.modelInstalled')}
/>
<Button
variant="ghost"
size="xs"
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
disabled={requesting}
onClick={() => { void request('DELETE'); }}
title={t('settings.voice.page.stt.modelDelete')}
aria-label={t('settings.voice.page.stt.modelDelete')}
>
<Icon name="delete-bin" className="h-4 w-4" />
</Button>
</>
) : model.downloading ? (
<span className="flex items-center gap-1.5">
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
<span className="typography-ui-compact tabular-nums text-muted-foreground">
{typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''}
</span>
</span>
) : (
<Button
variant="ghost"
size="xs"
className="h-6 w-6 p-0"
disabled={requesting}
onClick={() => { void request('POST'); }}
title={t('settings.voice.page.stt.modelDownload')}
aria-label={t('settings.voice.page.stt.modelDownload')}
>
<Icon name="download" className="h-4 w-4" />
</Button>
)}
{model.downloadError ? (
<span className="typography-meta text-[var(--status-error)]">{model.downloadError}</span>
) : null}
</div>
);
};
@@ -131,11 +389,6 @@ const OPENAI_VOICE_OPTIONS = [
export const VoiceSettings: React.FC = () => {
const { t } = useI18n();
const { isMobile } = useDeviceInfo();
const {
isSupported,
language,
setLanguage,
} = useBrowserVoice();
const voiceProvider = useConfigStore((state) => state.voiceProvider);
const setVoiceProvider = useConfigStore((state) => state.setVoiceProvider);
const speechRate = useConfigStore((state) => state.speechRate);
@@ -146,6 +399,22 @@ export const VoiceSettings: React.FC = () => {
const setSpeechVolume = useConfigStore((state) => state.setSpeechVolume);
const sayVoice = useConfigStore((state) => state.sayVoice);
const setSayVoice = useConfigStore((state) => state.setSayVoice);
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
const setLocalTtsVoiceId = useConfigStore((state) => state.setLocalTtsVoiceId);
const { speak: speakLocalTts, stop: stopLocalTts, isPlaying: isLocalTtsPlaying, error: localTtsError } = useLocalTTS();
const previewLocalVoice = useCallback(() => {
if (isLocalTtsPlaying) {
stopLocalTts();
return;
}
const voiceLabel = KOKORO_VOICE_OPTIONS.find((v) => v.id === localTtsVoiceId)?.label
?? String(localTtsVoiceId);
void speakLocalTts(t('settings.voice.page.preview.voiceLine', { voiceName: voiceLabel }), {
speakerId: localTtsVoiceId,
speed: useConfigStore.getState().speechRate,
});
}, [isLocalTtsPlaying, localTtsVoiceId, speakLocalTts, stopLocalTts, t]);
const browserVoice = useConfigStore((state) => state.browserVoice);
const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice);
const openaiVoice = useConfigStore((state) => state.openaiVoice);
@@ -172,19 +441,13 @@ export const VoiceSettings: React.FC = () => {
const setSttApiKey = useConfigStore((state) => state.setSttApiKey);
const sttModel = useConfigStore((state) => state.sttModel);
const setSttModel = useConfigStore((state) => state.setSttModel);
const wasmSttModel = useConfigStore((state) => state.wasmSttModel);
const setWasmSttModel = useConfigStore((state) => state.setWasmSttModel);
const sttLocalModel = useConfigStore((state) => state.sttLocalModel);
const setSttLocalModel = useConfigStore((state) => state.setSttLocalModel);
const sttLanguage = useConfigStore((state) => state.sttLanguage);
const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
const setSttSilenceThresholdDb = useConfigStore((state) => state.setSttSilenceThresholdDb);
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
const setSttSilenceHoldMs = useConfigStore((state) => state.setSttSilenceHoldMs);
const sttTranscribeOnStop = useConfigStore((state) => state.sttTranscribeOnStop);
const setSttTranscribeOnStop = useConfigStore((state) => state.setSttTranscribeOnStop);
const setShowMessageTTSButtons = useConfigStore((state) => state.setShowMessageTTSButtons);
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
const setVoiceModeEnabled = useConfigStore((state) => state.setVoiceModeEnabled);
const dictationEnabled = useConfigStore((state) => state.dictationEnabled);
const setDictationEnabled = useConfigStore((state) => state.setDictationEnabled);
const [isSayAvailable, setIsSayAvailable] = useState(false);
const [sayVoices, setSayVoices] = useState<Array<{ name: string; locale: string }>>([]);
@@ -274,7 +537,7 @@ export const VoiceSettings: React.FC = () => {
}, [isBrowserPreviewPlaying]);
useEffect(() => {
if (!voiceModeEnabled || (voiceProvider !== 'openai' && voiceProvider !== 'openai-compatible')) {
if (!showMessageTTSButtons || (voiceProvider !== 'openai' && voiceProvider !== 'openai-compatible')) {
setIsOpenAIAvailable(openaiApiKey.trim().length > 0);
return;
}
@@ -292,10 +555,10 @@ export const VoiceSettings: React.FC = () => {
};
checkOpenAIAvailability();
}, [openaiApiKey, voiceModeEnabled, voiceProvider]);
}, [openaiApiKey, showMessageTTSButtons, voiceProvider]);
useEffect(() => {
if (!voiceModeEnabled) {
if (!showMessageTTSButtons) {
setIsSayAvailable(false);
setSayVoices([]);
return;
@@ -317,7 +580,7 @@ export const VoiceSettings: React.FC = () => {
.catch(() => {
setIsSayAvailable(false);
});
}, [voiceModeEnabled]);
}, [showMessageTTSButtons]);
const previewVoice = useCallback(async () => {
if (previewAudio) {
@@ -498,11 +761,11 @@ export const VoiceSettings: React.FC = () => {
return (
<div className="space-y-8">
{/* Voice Setup */}
<div data-settings-item="voice.voice-setup" className="mb-8">
{/* Playback (read messages aloud) */}
<div data-settings-item="voice.playback" className="mb-8">
<div className="mb-1 px-1">
<h3 className="typography-ui-header font-medium text-foreground">
{t('settings.voice.page.section.voiceSetup')}
{t('settings.voice.page.section.playbackAndSummary')}
</h3>
</div>
@@ -512,15 +775,15 @@ export const VoiceSettings: React.FC = () => {
className="group flex cursor-pointer items-center gap-2 py-1.5"
role="button"
tabIndex={0}
aria-pressed={voiceModeEnabled}
onClick={() => setVoiceModeEnabled(!voiceModeEnabled)}
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setVoiceModeEnabled(!voiceModeEnabled); } }}
aria-pressed={showMessageTTSButtons}
onClick={() => setShowMessageTTSButtons(!showMessageTTSButtons)}
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setShowMessageTTSButtons(!showMessageTTSButtons); } }}
>
<Checkbox checked={voiceModeEnabled} onChange={setVoiceModeEnabled} ariaLabel={t('settings.voice.page.field.enableVoiceModeAria')} />
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.enableVoiceMode')}</span>
<Checkbox checked={showMessageTTSButtons} onChange={setShowMessageTTSButtons} ariaLabel={t('settings.voice.page.field.messageReadAloudButtonAria')} />
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.messageReadAloudButton')}</span>
</div>
{voiceModeEnabled && (
{showMessageTTSButtons && (
<>
<div className="pb-1.5 pt-0.5">
<div className="flex min-w-0 flex-col gap-1.5">
@@ -533,6 +796,7 @@ export const VoiceSettings: React.FC = () => {
<TooltipContent sideOffset={8} className="max-w-xs">
<ul className="space-y-1">
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.browser')}</li>
<li><strong>{t('settings.voice.page.provider.local')}</strong> {t('settings.voice.page.tooltip.localTts')}</li>
<li><strong>OpenAI:</strong> {t('settings.voice.page.tooltip.openai')}</li>
<li><strong>{t('settings.voice.page.provider.custom')}</strong> {t('settings.voice.page.tooltip.custom')}</li>
<li><strong>{t('settings.voice.page.provider.say')}</strong> {t('settings.voice.page.tooltip.say')}</li>
@@ -550,6 +814,15 @@ export const VoiceSettings: React.FC = () => {
>
{t('settings.voice.page.provider.browser')}
</Button>
<Button
variant="chip"
size="xs"
aria-pressed={voiceProvider === 'local'}
onClick={() => setVoiceProvider('local')}
className="!font-normal"
>
{t('settings.voice.page.provider.local')}
</Button>
<Button
variant="chip"
size="xs"
@@ -706,10 +979,39 @@ export const VoiceSettings: React.FC = () => {
</div>
)}
{/* Local (Kokoro) TTS model status */}
{voiceProvider === 'local' && <LocalTtsModelStatus />}
{/* Voice Selection */}
<div className="flex items-center gap-8 py-1.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.voice')}</span>
<div className="flex items-center gap-2 w-fit">
{voiceProvider === 'local' && (
<>
<Select
value={String(localTtsVoiceId)}
onValueChange={(value) => setLocalTtsVoiceId(Number.parseInt(value, 10) || 0)}
>
<SelectTrigger className="w-fit">
<SelectValue placeholder={t('settings.voice.page.field.selectVoicePlaceholder')}>
{(value) => KOKORO_VOICE_OPTIONS.find((v) => String(v.id) === value)?.label ?? value}
</SelectValue>
</SelectTrigger>
<SelectContent>
{KOKORO_VOICE_OPTIONS.map((v) => (
<SelectItem key={v.id} value={String(v.id)}>{v.label}</SelectItem>
))}
</SelectContent>
</Select>
<Button size="xs" variant="ghost" onClick={previewLocalVoice} title={t('settings.voice.page.actions.preview')}>
{isLocalTtsPlaying ? <Icon name="stop" className="w-3.5 h-3.5" /> : <Icon name="play" className="w-3.5 h-3.5" />}
</Button>
{localTtsError ? (
<span className="typography-meta text-[var(--status-error)]">{localTtsError}</span>
) : null}
</>
)}
{voiceProvider === 'openai' && isOpenAIAvailable && (
<>
<Select value={openaiVoice} onValueChange={setOpenaiVoice}>
@@ -775,7 +1077,7 @@ export const VoiceSettings: React.FC = () => {
<div className="flex items-center gap-8 py-1.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechRate')}</span>
<div className="flex items-center gap-2 w-fit">
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechRate} onChange={(e) => setSpeechRate(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechRate} onChange={(e) => setSpeechRate(Number(e.target.value))} className={sliderClass} />}
<NumberInput value={speechRate} onValueChange={setSpeechRate} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
</div>
</div>
@@ -784,7 +1086,7 @@ export const VoiceSettings: React.FC = () => {
<div className="flex items-center gap-8 py-1.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechPitch')}</span>
<div className="flex items-center gap-2 w-fit">
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechPitch} onChange={(e) => setSpeechPitch(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechPitch} onChange={(e) => setSpeechPitch(Number(e.target.value))} className={sliderClass} />}
<NumberInput value={speechPitch} onValueChange={setSpeechPitch} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
</div>
</div>
@@ -793,7 +1095,7 @@ export const VoiceSettings: React.FC = () => {
<div className="flex items-center gap-8 py-1.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechVolume')}</span>
<div className="flex items-center gap-2 w-fit">
{!isMobile && <input type="range" min={0} max={1} step={0.1} value={speechVolume} onChange={(e) => setSpeechVolume(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
{!isMobile && <input type="range" min={0} max={1} step={0.1} value={speechVolume} onChange={(e) => setSpeechVolume(Number(e.target.value))} className={sliderClass} />}
{isMobile ? (
<NumberInput value={Math.round(speechVolume * 100)} onValueChange={(v) => setSpeechVolume(v / 100)} min={0} max={100} step={10} className="w-16 tabular-nums" />
) : (
@@ -804,20 +1106,34 @@ export const VoiceSettings: React.FC = () => {
</div>
</div>
{/* Language */}
<div className="flex items-center gap-8 py-1.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.language')}</span>
<div className="flex items-center gap-2 w-fit">
<Select value={language} onValueChange={setLanguage} disabled={!isSupported}>
<SelectTrigger className="w-fit">
<SelectValue placeholder={t('settings.voice.page.field.selectLanguagePlaceholder')} />
</SelectTrigger>
<SelectContent>
{LANGUAGE_OPTIONS.map((lang) => (
<SelectItem key={lang.value} value={lang.value}>{lang.label}</SelectItem>
))}
</SelectContent>
</Select>
{/* TTS input mode */}
<div className="pb-1.5 pt-0.5">
<div className="flex min-w-0 flex-col gap-1.5">
<div className="flex items-center gap-1.5">
<span className="typography-ui-label text-foreground">
{t('settings.voice.page.field.ttsInputMode')}
</span>
</div>
<div className="flex flex-wrap items-center gap-1">
<Button
variant="chip"
size="xs"
aria-pressed={ttsInputMode === 'sanitized'}
onClick={() => setTtsInputMode('sanitized')}
className="!font-normal"
>
{t('settings.voice.page.field.ttsInputModeSanitized')}
</Button>
<Button
variant="chip"
size="xs"
aria-pressed={ttsInputMode === 'raw'}
onClick={() => setTtsInputMode('raw')}
className="!font-normal"
>
{t('settings.voice.page.field.ttsInputModeRaw')}
</Button>
</div>
</div>
</div>
</>
@@ -826,8 +1142,7 @@ export const VoiceSettings: React.FC = () => {
</div>
{/* Speech Recognition */}
{voiceModeEnabled && (
<div data-settings-item="voice.speech-recognition" className="mb-8">
<div data-settings-item="voice.speech-recognition" className="mb-8">
<div className="mb-1 px-1">
<h3 className="typography-ui-header font-medium text-foreground">
{t('settings.voice.page.section.speechRecognition')}
@@ -835,6 +1150,19 @@ export const VoiceSettings: React.FC = () => {
</div>
<section className="px-2 pb-2 pt-0 space-y-0">
<div
className="group flex cursor-pointer items-center gap-2 py-1.5"
role="button"
tabIndex={0}
aria-pressed={dictationEnabled}
onClick={() => setDictationEnabled(!dictationEnabled)}
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setDictationEnabled(!dictationEnabled); } }}
>
<Checkbox checked={dictationEnabled} onChange={setDictationEnabled} ariaLabel={t('settings.voice.page.field.enableVoiceInputAria')} />
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.enableVoiceInput')}</span>
</div>
{dictationEnabled && (<>
<div className="pb-1.5 pt-0.5">
<div className="flex min-w-0 flex-col gap-1.5">
<div className="flex items-center gap-1.5">
@@ -845,7 +1173,7 @@ export const VoiceSettings: React.FC = () => {
</TooltipTrigger>
<TooltipContent sideOffset={8} className="max-w-xs">
<ul className="space-y-1">
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.sttBrowser')}</li>
<li><strong>{t('settings.voice.page.provider.local')}</strong> {t('settings.voice.page.tooltip.sttLocal')}</li>
<li><strong>{t('settings.voice.page.provider.server')}</strong> {t('settings.voice.page.tooltip.sttServer')}</li>
</ul>
</TooltipContent>
@@ -855,53 +1183,34 @@ export const VoiceSettings: React.FC = () => {
<Button
variant="chip"
size="xs"
aria-pressed={sttProvider === 'browser'}
onClick={() => setSttProvider('browser')}
aria-pressed={sttProvider === 'local'}
onClick={() => setSttProvider('local')}
className="!font-normal"
>
{t('settings.voice.page.provider.browser')}
{t('settings.voice.page.provider.local')}
</Button>
<Button
variant="chip"
size="xs"
aria-pressed={sttProvider === 'server'}
onClick={() => setSttProvider('server')}
aria-pressed={sttProvider === 'openai-compatible'}
onClick={() => setSttProvider('openai-compatible')}
className="!font-normal"
>
{t('settings.voice.page.provider.server')}
</Button>
<Button
variant="chip"
size="xs"
aria-pressed={sttProvider === 'wasm'}
onClick={() => setSttProvider('wasm')}
className="!font-normal"
>
{t('settings.voice.page.provider.wasm')}
</Button>
</div>
</div>
</div>
{sttProvider === 'server' && (
<div className="py-1.5 space-y-2">
<div
className="group flex cursor-pointer items-center gap-2 py-1.5"
role="button"
tabIndex={0}
aria-pressed={sttTranscribeOnStop}
onClick={() => setSttTranscribeOnStop(!sttTranscribeOnStop)}
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setSttTranscribeOnStop(!sttTranscribeOnStop); } }}
>
<Checkbox checked={sttTranscribeOnStop} onChange={setSttTranscribeOnStop} ariaLabel={t('settings.voice.page.field.transcribeOnStopAria')} />
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.transcribeOnStop')}</span>
</div>
{sttProvider === 'local' && (
<div className="py-1.5">
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.model')}</span>
<LocalModelPicker selectedModelId={sttLocalModel} onSelect={setSttLocalModel} />
</div>
)}
{!audioStreamService.isSupported() && (
<p className="typography-meta text-[var(--status-error)]">
{t('settings.voice.page.field.sttBrowserSupportError')}
</p>
)}
{sttProvider === 'openai-compatible' && (
<div className="py-1.5 space-y-2">
<div>
<span className={cn("typography-ui-label text-foreground", !sttServerUrl.trim() && "text-[var(--status-error)]")}>
{t('settings.voice.page.field.serverUrl')}
@@ -979,129 +1288,14 @@ export const VoiceSettings: React.FC = () => {
/>
</div>
</div>
<div className="flex items-center gap-8 py-0.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.silenceThreshold')}</span>
<div className="flex items-center gap-2 w-fit">
{!isMobile && <input type="range" min={-60} max={-20} step={1} value={sttSilenceThresholdDb} onChange={(e) => setSttSilenceThresholdDb(Number(e.target.value))} className={sliderClass} />}
<span className="typography-ui-label text-foreground tabular-nums min-w-[3.5rem] text-right">
{sttSilenceThresholdDb} dB
</span>
</div>
</div>
<div className="flex items-center gap-8 py-0.5">
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.silenceHold')}</span>
<div className="flex items-center gap-2 w-fit">
{!isMobile && <input type="range" min={500} max={3000} step={100} value={sttSilenceHoldMs} onChange={(e) => setSttSilenceHoldMs(Number(e.target.value))} className={sliderClass} />}
<NumberInput value={sttSilenceHoldMs} onValueChange={setSttSilenceHoldMs} min={500} max={3000} step={100} className="w-20 tabular-nums" />
<span className="typography-meta text-muted-foreground">{t('settings.voice.page.field.millisecondsUnit')}</span>
</div>
</div>
</div>
)}
{sttProvider === 'wasm' && (
<div className="py-1.5 space-y-2">
<div>
<span className="typography-ui-label text-muted-foreground">
{t('settings.voice.page.stt.wasmModel')}
</span>
<Select value={wasmSttModel} onValueChange={setWasmSttModel}>
<SelectTrigger className="mt-0.5">
<SelectValue placeholder={t('settings.voice.page.stt.wasmModel')} />
</SelectTrigger>
<SelectContent>
{WASM_MODELS.map((m) => (
<SelectItem key={m.id} value={m.id}>
<div className="flex flex-col">
<span>{m.name}</span>
<span className="typography-ui-compact text-muted-foreground">
{m.size} · {m.languages}
</span>
</div>
</SelectItem>
))}
</SelectContent>
</Select>
<p className="typography-ui-compact text-muted-foreground mt-0.5">
{WASM_MODELS.find((m) => m.id === wasmSttModel)?.description}
</p>
</div>
{/* Model status indicator */}
<WasmModelStatusIndicator modelId={wasmSttModel} />
</div>
)}
</>)}
</section>
</div>
)}
{/* Playback & Summarization */}
<div data-settings-item="voice.playback" className="mb-8">
<div className="mb-1 px-1">
<h3 className="typography-ui-header font-medium text-foreground">
{t('settings.voice.page.section.playbackAndSummary')}
</h3>
</div>
<section className="px-2 pb-2 pt-0 space-y-0">
<div
className="group flex cursor-pointer items-center gap-2 py-1.5"
role="button"
tabIndex={0}
aria-pressed={showMessageTTSButtons}
onClick={() => setShowMessageTTSButtons(!showMessageTTSButtons)}
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setShowMessageTTSButtons(!showMessageTTSButtons); } }}
>
<Checkbox checked={showMessageTTSButtons} onChange={setShowMessageTTSButtons} ariaLabel={t('settings.voice.page.field.messageReadAloudButtonAria')} />
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.messageReadAloudButton')}</span>
</div>
<div className="pb-1.5 pt-0.5">
<div className="flex min-w-0 flex-col gap-1.5">
<div className="flex items-center gap-1.5">
<span className="typography-ui-label text-foreground">
{t('settings.voice.page.field.ttsInputMode')}
</span>
</div>
<div className="flex flex-wrap items-center gap-1">
<Button
variant="chip"
size="xs"
aria-pressed={ttsInputMode === 'sanitized'}
onClick={() => setTtsInputMode('sanitized')}
className="!font-normal"
>
{t('settings.voice.page.field.ttsInputModeSanitized')}
</Button>
<Button
variant="chip"
size="xs"
aria-pressed={ttsInputMode === 'raw'}
onClick={() => setTtsInputMode('raw')}
className="!font-normal"
>
{t('settings.voice.page.field.ttsInputModeRaw')}
</Button>
</div>
</div>
</div>
</section>
{voiceModeEnabled && isSupported && (
<div className="mt-2 px-2">
<p className="typography-meta text-muted-foreground">
{t('settings.voice.page.hint.shiftClickPrefix')}
{' '}
<kbd className="px-1 py-0.5 mx-0.5 rounded border border-[var(--interactive-border)] bg-background typography-mono text-[10px]">Shift</kbd>
{' + '}
<kbd className="px-1 py-0.5 mx-0.5 rounded border border-[var(--interactive-border)] bg-background typography-mono text-[10px]">Click</kbd>
{' '}
{t('settings.voice.page.hint.shiftClickSuffix')}
</p>
</div>
)}
</div>
</div>
);
};
@@ -1030,7 +1030,7 @@ export const SettingsView: React.FC<SettingsViewProps> = ({ onClose, forceMobile
: <Icon name={iconName!} className="h-4 w-4 shrink-0" />}
<span className="flex items-center gap-1.5 whitespace-nowrap overflow-hidden transition-opacity duration-150 opacity-100">
<span className="typography-ui-label font-normal truncate">{getPageTitle(page.slug)}</span>
{(page.slug === 'voice' || page.slug === 'tunnel') && (
{page.slug === 'tunnel' && (
<span className="shrink-0 typography-micro px-1 rounded leading-none pb-px text-[var(--status-warning)] bg-[var(--status-warning)]/10">
{t('settings.view.badge.beta')}
</span>
@@ -1,392 +0,0 @@
/**
* BrowserVoiceButton Component
*
* Voice toggle button for browser-based voice chat with language selection.
* Shows visual state indicators for different voice modes.
*
* @example
* ```tsx
* <BrowserVoiceButton />
* ```
*/
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
import { useConfigStore } from '@/stores/useConfigStore';
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
import { isVSCodeRuntime } from '@/lib/desktop';
import { Button } from '@/components/ui/button';
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from '@/components/ui/tooltip';
import { VoiceStatusIndicator } from './VoiceStatusIndicator';
import { toast } from '@/components/ui/toast';
import { Icon } from "@/components/icon/Icon";
import { useI18n } from '@/lib/i18n';
// Status text for accessibility and labels
const statusLabels: Record<string, string> = {
idle: 'Start Voice',
listening: 'Listening',
processing: 'Processing',
speaking: 'AI Speaking',
error: 'Voice Error',
};
// iOS Safari detection utility
const isIOSSafari = (): boolean => {
if (typeof navigator === 'undefined') return false;
const userAgent = navigator.userAgent.toLowerCase();
const isIOS = /iphone|ipad|ipod/i.test(userAgent);
const isSafari = /safari/i.test(userAgent) && !/chrome|crios|crmo/i.test(userAgent);
return isIOS && isSafari;
};
const normalizeVoiceErrorMessage = (error: string): string => {
const isMediaDevicesError =
error.includes('getUserMedia') ||
error.includes('mediaDevices') ||
error.includes('Cannot read properties of undefined');
if (!isMediaDevicesError) {
return error;
}
if (typeof window !== 'undefined' && !window.isSecureContext) {
return 'Voice requires a secure connection (HTTPS) or localhost. Please use HTTPS or access via localhost.';
}
return 'Microphone access is unavailable in this runtime. On desktop, check System Settings -> Privacy & Security -> Microphone for OpenChamber.';
};
/**
* Browser Voice Button with language selection
*/
export function BrowserVoiceButton() {
const { t } = useI18n();
const voiceModeEnabled = useConfigStore((s) => s.voiceModeEnabled);
const sttProvider = useConfigStore((s) => s.sttProvider);
const sttTranscribeOnStop = useConfigStore((s) => s.sttTranscribeOnStop);
const {
status,
isSupported,
error,
startVoice,
stopVoice,
finishVoiceInput,
conversationMode,
toggleConversationMode,
isMobile,
} = useBrowserVoice();
const [isPressing, setIsPressing] = useState(false);
const isVSCode = isVSCodeRuntime();
const buttonSizeClass = isMobile ? 'h-8 w-8 min-h-[32px] min-w-[32px]' : (isVSCode ? 'h-5 w-5' : 'h-6 w-6');
const iconSizeClass = isMobile ? 'h-[18px] w-[18px]' : (isVSCode ? 'h-4 w-4' : 'h-[18px] w-[18px]');
const continuousIconSizeClass = 'size-[18px]';
const clearHoverBackgroundClass = 'bg-transparent hover:!bg-transparent active:!bg-transparent focus-visible:!bg-transparent';
// Refs for touch handling
const touchHandledRef = useRef(false);
const isIOSSafariRef = useRef(false);
const longPressTimerRef = useRef<NodeJS.Timeout | null>(null);
const longPressTriggeredRef = useRef(false);
const lastToastedErrorRef = useRef<string | null>(null);
// Initialize iOS detection on mount
useEffect(() => {
isIOSSafariRef.current = isIOSSafari();
}, []);
// NOTE: Do NOT pre-request microphone permission on mount.
// Permission is requested when the user explicitly taps the mic button.
// Pre-requesting causes an unwanted permission prompt on mobile page load.
// Determine active states
const isActive = status === 'listening' || status === 'speaking' || status === 'processing';
const isError = status === 'error';
const isIdle = status === 'idle';
const isSpeaking = status === 'speaking';
// WASM STT always needs finishVoiceInput to flush the recorder and transcribe.
// Server STT uses it when sttTranscribeOnStop is enabled.
const canTranscribeOnStop = sttProvider === 'wasm' || (sttProvider === 'server' && sttTranscribeOnStop);
const isListeningWithTranscribeOnStop = status === 'listening' && canTranscribeOnStop;
// Show toast notification when voice error occurs
useEffect(() => {
if (isError && error) {
if (lastToastedErrorRef.current === error) {
return;
}
lastToastedErrorRef.current = error;
const displayError = normalizeVoiceErrorMessage(error);
toast.error(displayError, {
duration: 5000,
});
}
if (!isError) {
lastToastedErrorRef.current = null;
}
}, [isError, error]);
// Status text for accessibility
const statusText = isError
? error || 'Voice Error'
: isListeningWithTranscribeOnStop
? t('voice.action.finishAndTranscribe')
: conversationMode && status === 'idle'
? 'Start Voice (Continuous mode on)'
: statusLabels[status] || 'Start Voice';
// Tooltip content based on state
const getTooltipContent = () => {
if (isError && error) {
return normalizeVoiceErrorMessage(error);
}
if (isListeningWithTranscribeOnStop) {
return t('voice.action.finishAndTranscribe');
}
if (isActive) {
return 'Stop voice conversation';
}
if (isMobile) {
return 'Start voice conversation';
}
return `Start voice conversation (Shift+Click for continuous mode) • Cmd/Ctrl+Shift+V to toggle`;
};
// Handle voice activation (used by both click and touch)
const activateVoice = useCallback(async () => {
if (isActive) {
if (status === 'listening' && canTranscribeOnStop) {
finishVoiceInput();
return;
}
stopVoice();
} else if (status !== 'error') {
// On mobile, we must NOT do any async operations before calling startVoice()
// because iOS Safari requires SpeechRecognition.start() to be called
// synchronously within the user gesture handler
if (isMobile) {
// Start voice immediately - no await before this!
// Audio unlock is now handled inside startVoice() for mobile
startVoice();
} else {
// Desktop can use async path
try {
await startVoice();
} catch (err) {
console.error('Failed to start voice:', err);
}
}
} else {
// Reset from error state
if (isMobile) {
startVoice();
} else {
try {
await startVoice();
} catch (err) {
console.error('Failed to start voice:', err);
}
}
}
}, [isActive, status, canTranscribeOnStop, finishVoiceInput, startVoice, stopVoice, isMobile]);
// Handle Shift+Click to toggle conversation mode
const handleClick = useCallback(async (e: React.MouseEvent) => {
// Prevent double-firing if touch already handled this
if (touchHandledRef.current) {
touchHandledRef.current = false;
return;
}
// Shift+Click toggles conversation mode
if (e.shiftKey) {
toggleConversationMode();
return;
}
await activateVoice();
}, [activateVoice, toggleConversationMode]);
// Handle touch start for mobile devices
const handleTouchStart = useCallback((e: React.TouchEvent) => {
// Prevent default to stop mouse event emulation
e.preventDefault();
// Mark that touch handled this interaction
touchHandledRef.current = true;
longPressTriggeredRef.current = false;
// Immediate visual feedback
setIsPressing(true);
// Set up long-press timer for toggling conversation mode (500ms)
longPressTimerRef.current = setTimeout(() => {
longPressTriggeredRef.current = true;
toggleConversationMode();
// Haptic feedback if available
if (navigator.vibrate) {
navigator.vibrate(50);
}
setIsPressing(false);
}, 500);
}, [toggleConversationMode]);
// Handle touch end
const handleTouchEnd = useCallback(() => {
// Clear long-press timer
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
longPressTimerRef.current = null;
}
// Only activate voice if long-press wasn't triggered
if (!longPressTriggeredRef.current) {
activateVoice();
}
setIsPressing(false);
}, [activateVoice]);
// Handle touch cancel
const handleTouchCancel = useCallback(() => {
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
longPressTimerRef.current = null;
}
setIsPressing(false);
}, []);
const handleToggleConversationMode = useCallback((event: React.MouseEvent<HTMLButtonElement>) => {
event.preventDefault();
event.stopPropagation();
toggleConversationMode();
}, [toggleConversationMode]);
// If voice mode is disabled, don't render anything
if (!voiceModeEnabled) {
return null;
}
// If not supported, show disabled button with tooltip
if (!isSupported) {
const supportDetails = browserVoiceService.getSupportDetails();
const tooltipMessage = !supportDetails.secureContext
? 'Voice requires HTTPS or localhost. Please use a secure connection.'
: !supportDetails.recognition
? 'Speech recognition not supported in this browser. Try Chrome, Edge, or Safari.'
: !supportDetails.synthesis
? 'Speech synthesis not supported in this browser.'
: 'Voice not supported in this browser';
return (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
size="icon"
variant="ghost"
disabled
aria-label={tooltipMessage}
className={`${buttonSizeClass} p-0 ${clearHoverBackgroundClass}`}
>
<Icon name="mic-off" className={`${iconSizeClass} opacity-50`} />
</Button>
</TooltipTrigger>
<TooltipContent side="bottom" align="center">
<p className="max-w-[200px] text-center">{tooltipMessage}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
);
}
return (
<div className={`flex items-center ${isMobile ? 'gap-1' : 'gap-1.5'}`}>
{/* Status indicator with label - show when active, simplified on mobile */}
{isActive && !isMobile && (
<VoiceStatusIndicator
status={status}
showLabel
size="sm"
className="mr-1"
/>
)}
{/* Voice button with tooltip */}
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
size="icon"
variant="ghost"
onClick={handleClick}
onTouchStart={handleTouchStart}
onTouchEnd={handleTouchEnd}
onTouchCancel={handleTouchCancel}
aria-label={statusText}
className={`
relative
${buttonSizeClass}
p-0
${clearHoverBackgroundClass}
touch-manipulation
${isPressing ? 'scale-95 opacity-80' : ''}
${conversationMode && isIdle && isMobile ? 'ring-1 ring-primary/50' : ''}
`}
style={{
WebkitTapHighlightColor: 'transparent',
touchAction: 'manipulation',
}}
>
{isActive ? (
isSpeaking ? (
// Green speaker icon when AI is speaking
<Icon name="volume-up" className={`${iconSizeClass} text-green-400 animate-pulse`} />
) : (
// Red stop icon for listening/processing (both mobile and desktop)
<Icon name="stop-circle" className={`${iconSizeClass} text-[var(--status-error)]`} />
)
) : (
<VoiceStatusIndicator
status={isError ? 'idle' : status}
size={isMobile || isVSCode ? 'sm' : 'md'}
/>
)}
</Button>
</TooltipTrigger>
<TooltipContent side="bottom" align="center">
<p className="max-w-[200px] text-center">{getTooltipContent()}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
{/* Conversation mode toggle button */}
{(status === 'idle' || status === 'error') && (
<Button
size="icon"
variant="ghost"
onPointerDownCapture={(event) => event.stopPropagation()}
onClick={handleToggleConversationMode}
aria-label={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
title={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
className={
`${buttonSizeClass} p-0 ${clearHoverBackgroundClass} ${conversationMode ? 'text-[var(--status-info)] hover:text-[var(--status-info)]' : 'text-muted-foreground hover:text-foreground'}`
}
>
<Icon name="voice-recognition" className={continuousIconSizeClass} />
</Button>
)}
</div>
);
}
@@ -1,30 +0,0 @@
import React from 'react';
import { useVoiceContext } from '@/hooks/useVoiceContext';
import { useConfigStore } from '@/stores/useConfigStore';
const VoiceContextBridge = React.memo(function VoiceContextBridge() {
useVoiceContext();
return null;
});
/**
* Provider component that initializes voice context sync.
* Wrap the app with this to enable voice session awareness.
*
* @example
* ```tsx
* <VoiceProvider>
* <App />
* </VoiceProvider>
* ```
*/
export function VoiceProvider({ children }: { children: React.ReactNode }) {
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
return (
<>
{voiceModeEnabled ? <VoiceContextBridge /> : null}
{children}
</>
);
}
@@ -1,139 +0,0 @@
/**
* VoiceStatusIndicator Component
*
* Reusable visual indicator for voice mode states with icons, animations,
* and optional status text labels.
*
* @example
* ```tsx
* // Basic usage - icon only
* <VoiceStatusIndicator status="listening" />
*
* // With label
* <VoiceStatusIndicator status="listening" showLabel />
*
* // Different size
* <VoiceStatusIndicator status="processing" size="lg" />
* ```
*/
import React from 'react';
import type { BrowserVoiceStatus } from '@/hooks/useBrowserVoice';
import { useI18n } from '@/lib/i18n';
import { Icon } from "@/components/icon/Icon";
import type { IconName } from "@/components/icon/icons";
export interface VoiceStatusIndicatorProps {
/** Current voice status */
status: BrowserVoiceStatus;
/** Show text label next to icon */
showLabel?: boolean;
/** Size of the indicator */
size?: 'sm' | 'md' | 'lg';
/** Optional className for styling */
className?: string;
/** Whether conversation mode is active (shows indicator dot when idle) */
conversationMode?: boolean;
}
const sizeClasses = {
sm: {
icon: 'w-4 h-4',
container: 'gap-1.5',
},
md: {
icon: 'w-5 h-5',
container: 'gap-2',
},
lg: {
icon: 'w-6 h-6',
container: 'gap-2.5',
},
};
const statusConfig: Record<
BrowserVoiceStatus,
{
icon: IconName;
color: string;
labelKey:
| 'voice.status.idle'
| 'voice.status.listening'
| 'voice.status.processing'
| 'voice.status.speaking'
| 'voice.status.error';
animation?: string;
}
> = {
idle: {
icon: "mic-off",
color: 'text-muted-foreground',
labelKey: 'voice.status.idle',
},
listening: {
icon: "mic",
color: 'text-primary',
labelKey: 'voice.status.listening',
animation: 'animate-pulse',
},
processing: {
icon: "loader-4",
color: 'text-primary',
labelKey: 'voice.status.processing',
animation: 'animate-spin',
},
speaking: {
icon: "volume-up",
color: 'text-green-500',
labelKey: 'voice.status.speaking',
},
error: {
icon: "alert",
color: 'text-destructive',
labelKey: 'voice.status.error',
},
};
/**
* VoiceStatusIndicator - Visual indicator for voice mode states
*/
export function VoiceStatusIndicator({
status,
showLabel = false,
size = 'md',
className = '',
conversationMode = false,
}: VoiceStatusIndicatorProps) {
const { t } = useI18n();
const config = statusConfig[status];
const statusIconName = config.icon;
const sizeClass = sizeClasses[size];
const containerClass = showLabel ? sizeClass.container : '';
return (
<div className={`flex items-center ${containerClass} ${className}`}>
<div className="relative">
<Icon name={statusIconName}
className={`
${sizeClass.icon}
${config.color}
${config.animation || ''}
`}
aria-hidden="true"
/>
{/* Conversation mode indicator dot - only when idle and conversation mode is on */}
{conversationMode && status === 'idle' && (
<span
className="absolute -top-0.5 -right-0.5 w-2 h-2 bg-green-500 rounded-full"
aria-label={t('voice.status.conversationModeActiveAria')}
/>
)}
</div>
{showLabel && (
<span className={`typography-meta ${config.color}`}>
{t(config.labelKey)}
</span>
)}
</div>
);
}
@@ -1,2 +0,0 @@
export { VoiceProvider } from './VoiceProvider';
export { BrowserVoiceButton } from './BrowserVoiceButton';
File diff suppressed because it is too large Load Diff
+408
View File
@@ -0,0 +1,408 @@
/**
* Streaming dictation state machine.
*
* Status flow: idle -> recording -> uploading -> idle | failed.
* While recording, mic PCM chunks stream to the server, which sends back live
* partial transcripts. Confirm finalizes and resolves the full text; failed
* dictations retain their audio segments so retry can replay them.
*/
import { useCallback, useEffect, useRef, useState } from 'react';
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
import { useConfigStore } from '@/stores/useConfigStore';
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
export interface UseDictationOptions {
onTranscript?: (text: string) => void;
onError?: (error: Error) => void;
canStart?: () => boolean;
}
export interface UseDictationResult {
status: DictationStatus;
isRecording: boolean;
isProcessing: boolean;
partialTranscript: string;
volume: number;
duration: number;
error: string | null;
errorReason: string | null;
startDictation: () => Promise<void>;
confirmDictation: () => Promise<string | null>;
cancelDictation: () => Promise<void>;
retryFailedDictation: () => Promise<string | null>;
acceptPartialTranscript: () => string | null;
discardFailedDictation: () => void;
}
const DURATION_TICK_MS = 1000;
const toError = (value: unknown): Error =>
value instanceof Error ? value : new Error(String(value));
const getDictationStartOptions = (): DictationStartOptions => {
const state = useConfigStore.getState();
const language = state.sttLanguage?.trim();
if (state.sttProvider === 'openai-compatible') {
return {
provider: 'openai-compatible',
...(language ? { language } : {}),
openaiCompatible: {
baseUrl: state.sttServerUrl,
model: state.sttModel,
...(state.sttApiKey ? { apiKey: state.sttApiKey } : {}),
},
};
}
return {
provider: 'local',
...(language ? { language } : {}),
localModel: state.sttLocalModel,
};
};
export function useDictation(options: UseDictationOptions = {}): UseDictationResult {
const { onTranscript, onError, canStart } = options;
const [status, setStatus] = useState<DictationStatus>('idle');
const [partialTranscript, setPartialTranscript] = useState('');
const [duration, setDuration] = useState(0);
const [error, setError] = useState<string | null>(null);
const [errorReason, setErrorReason] = useState<string | null>(null);
const statusRef = useRef(status);
useEffect(() => {
statusRef.current = status;
}, [status]);
const latestPartialRef = useRef('');
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const actionGateRef = useRef({ starting: false, confirming: false, cancelling: false });
const onTranscriptRef = useRef(onTranscript);
const onErrorRef = useRef(onError);
useEffect(() => {
onTranscriptRef.current = onTranscript;
onErrorRef.current = onError;
}, [onTranscript, onError]);
const senderRef = useRef<DictationStreamSender | null>(null);
if (!senderRef.current) {
senderRef.current = new DictationStreamSender({
client: dictationClient,
getStartOptions: getDictationStartOptions,
});
}
const stopDurationTracking = useCallback(() => {
if (durationIntervalRef.current) {
clearInterval(durationIntervalRef.current);
durationIntervalRef.current = null;
}
}, []);
const startDurationTracking = useCallback(() => {
if (durationIntervalRef.current) {
return;
}
durationIntervalRef.current = setInterval(() => {
setDuration((prev) => prev + 1);
}, DURATION_TICK_MS);
}, []);
const reportError = useCallback((err: unknown) => {
const normalized = toError(err);
setError(normalized.message);
const reason = (normalized as Error & { reasonCode?: string }).reasonCode;
setErrorReason(typeof reason === 'string' ? reason : null);
onErrorRef.current?.(normalized);
}, []);
const clearError = useCallback(() => {
setError(null);
setErrorReason(null);
}, []);
const clearStreamingState = useCallback(() => {
senderRef.current?.clearAll();
latestPartialRef.current = '';
setPartialTranscript('');
}, []);
// Live partial transcripts for the active dictation.
useEffect(() => {
return dictationClient.onPartial((dictationId, text) => {
const activeDictationId = senderRef.current?.getDictationId();
if (!activeDictationId || dictationId !== activeDictationId) {
return;
}
latestPartialRef.current = text;
setPartialTranscript(text);
});
}, []);
// Restart the stream (replaying buffered segments) after a reconnect.
useEffect(() => {
return dictationClient.subscribeConnectionStatus((connected) => {
if (!connected) {
return;
}
if (statusRef.current !== 'recording') {
return;
}
void senderRef.current?.restartStream().catch((err) => {
reportError(err);
});
});
}, [reportError]);
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
senderRef.current?.enqueueSegment(audioData);
},
onError: (err) => {
onErrorRef.current?.(err);
},
});
const audioStopRef = useRef(audio.stop);
useEffect(() => {
audioStopRef.current = audio.stop;
}, [audio.stop]);
const handleSuccess = useCallback(
(text: string): string | null => {
setDuration(0);
setStatus('idle');
const transcriptText = text.trim().length > 0 ? text.trim() : latestPartialRef.current.trim();
clearStreamingState();
if (!transcriptText) {
return null;
}
onTranscriptRef.current?.(transcriptText);
return transcriptText;
},
[clearStreamingState],
);
const handleFailure = useCallback(
(failure: unknown) => {
if (senderRef.current?.hasSegments()) {
setStatus('failed');
} else {
setStatus('idle');
}
reportError(failure);
},
[reportError],
);
const startDictation = useCallback(async () => {
const gate = actionGateRef.current;
if (gate.starting || gate.confirming || gate.cancelling) {
return;
}
if (statusRef.current !== 'idle') {
return;
}
if (canStart && !canStart()) {
return;
}
gate.starting = true;
clearError();
setPartialTranscript('');
setDuration(0);
setStatus('recording');
statusRef.current = 'recording';
clearStreamingState();
try {
await audio.start();
startDurationTracking();
// Open the stream eagerly so partials start flowing immediately.
await senderRef.current?.restartStream().catch((err) => {
// Non-fatal: segments buffer locally and finish() retries the
// start, but surface the reason (e.g. model downloading) so
// the overlay can show it.
reportError(err);
});
} catch (err) {
await audio.stop().catch(() => undefined);
stopDurationTracking();
setStatus('idle');
statusRef.current = 'idle';
reportError(err);
} finally {
gate.starting = false;
}
}, [audio, canStart, clearError, clearStreamingState, reportError, startDurationTracking, stopDurationTracking]);
const cancelDictation = useCallback(async () => {
const gate = actionGateRef.current;
if (gate.cancelling) {
return;
}
if (statusRef.current !== 'recording' && statusRef.current !== 'uploading') {
return;
}
gate.cancelling = true;
stopDurationTracking();
setDuration(0);
clearError();
// Optimistic: dismiss the overlay immediately. Tearing down the audio
// graph (AudioContext.close) can take up to ~1s on mobile WebViews and
// must not delay the visible response to Cancel.
setStatus('idle');
statusRef.current = 'idle';
try {
senderRef.current?.cancel();
} catch {
// no-op
}
clearStreamingState();
try {
await audio.stop();
} catch {
// Cancelled anyway; mic teardown failures are not user-actionable.
} finally {
gate.cancelling = false;
}
}, [audio, clearError, clearStreamingState, stopDurationTracking]);
const confirmDictation = useCallback(async (): Promise<string | null> => {
const gate = actionGateRef.current;
if (gate.confirming) {
return null;
}
if (statusRef.current !== 'recording') {
return null;
}
gate.confirming = true;
clearError();
stopDurationTracking();
try {
await audio.stop();
setStatus('uploading');
statusRef.current = 'uploading';
const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
if (finalSeq < 0) {
return handleSuccess('');
}
const result = await senderRef.current!.finish(finalSeq);
return handleSuccess(result.text);
} catch (err) {
handleFailure(err);
return null;
} finally {
gate.confirming = false;
}
}, [audio, clearError, handleFailure, handleSuccess, stopDurationTracking]);
const retryFailedDictation = useCallback(async (): Promise<string | null> => {
if (statusRef.current !== 'failed' || !senderRef.current?.hasSegments()) {
return null;
}
clearError();
setStatus('uploading');
statusRef.current = 'uploading';
try {
senderRef.current.resetStreamForReplay();
const finalSeq = senderRef.current.getFinalSeq();
const result = await senderRef.current.finish(finalSeq);
return handleSuccess(result.text);
} catch (err) {
handleFailure(err);
return null;
}
}, [clearError, handleFailure, handleSuccess]);
/**
* Failed dictations still hold the last streamed partial transcript.
* Accept it as-is instead of retrying the full transcription.
*/
const acceptPartialTranscript = useCallback((): string | null => {
if (statusRef.current !== 'failed') {
return null;
}
const text = latestPartialRef.current.trim();
setDuration(0);
setStatus('idle');
statusRef.current = 'idle';
clearError();
try {
senderRef.current?.cancel();
} catch {
// no-op
}
clearStreamingState();
if (!text) {
return null;
}
onTranscriptRef.current?.(text);
return text;
}, [clearError, clearStreamingState]);
const discardFailedDictation = useCallback(() => {
setDuration(0);
setStatus('idle');
statusRef.current = 'idle';
clearError();
clearStreamingState();
}, [clearError, clearStreamingState]);
// While recording without an open stream (e.g. the model is still
// downloading), retry the stream start so live partials kick in as soon
// as the provider becomes ready. Buffered segments replay on success.
useEffect(() => {
if (status !== 'recording' || errorReason !== 'model_download_in_progress') {
return;
}
const interval = setInterval(() => {
if (statusRef.current !== 'recording' || senderRef.current?.getDictationId()) {
return;
}
senderRef.current?.restartStream().then(() => {
clearError();
}).catch((err) => {
reportError(err);
});
}, 3000);
return () => clearInterval(interval);
}, [status, errorReason, clearError, reportError]);
useEffect(() => {
return () => {
stopDurationTracking();
void audioStopRef.current().catch(() => undefined);
senderRef.current?.cancel();
};
}, [stopDurationTracking]);
return {
status,
isRecording: status === 'recording',
isProcessing: status === 'uploading',
partialTranscript,
volume: audio.volume,
duration,
error,
errorReason,
startDictation,
confirmDictation,
cancelDictation,
retryFailedDictation,
acceptPartialTranscript,
discardFailedDictation,
};
}
@@ -474,6 +474,18 @@ export const useKeyboardShortcuts = () => {
return;
}
if (eventMatchesShortcut(e, combo('toggle_dictation'))) {
const { activeMainTab, isCommandPaletteOpen, isHelpDialogOpen, isSessionSwitcherOpen, isSettingsDialogOpen } = useUIStore.getState();
if (activeMainTab !== 'chat' || isCommandPaletteOpen || isHelpDialogOpen || isSessionSwitcherOpen || isSettingsDialogOpen) {
return;
}
e.preventDefault();
// Dictation state lives inside the composer's isolated component;
// toggle it via an event instead of subscribing this hot hook to it.
window.dispatchEvent(new CustomEvent('openchamber:dictation-toggle'));
return;
}
if (e.key === 'Escape') {
const target = e.target as Element | null;
const isInsideDialog = Boolean(target?.closest('[role="dialog"]'));
+266
View File
@@ -0,0 +1,266 @@
/**
* useLocalTTS Hook
*
* React hook for local (Kokoro via sherpa-onnx) text-to-speech playback.
* Synthesis runs on the OpenChamber server in the dictation worker.
*
* Long texts are pipelined by sentence chunks: the first chunk starts playing
* as soon as it is synthesized while the next chunk synthesizes in the
* background, so time-to-first-audio stays ~1 chunk regardless of message
* length.
*/
import { useCallback, useEffect, useRef, useState } from 'react';
import { runtimeFetch } from '@/lib/runtime-fetch';
export interface LocalTTSSpeakOptions {
/** Kokoro speaker id (0-10) */
speakerId?: number;
/** Playback speed multiplier (1.0 = normal) */
speed?: number;
onStart?: () => void;
onEnd?: () => void;
onError?: (error: string) => void;
}
export interface UseLocalTTSReturn {
isPlaying: boolean;
error: string | null;
speak: (text: string, options?: LocalTTSSpeakOptions) => Promise<void>;
stop: () => void;
/** Unlock audio for mobile Safari - call this on user gesture */
unlockAudio: () => Promise<void>;
}
/** Target chunk size: big enough to amortize requests, small enough for low latency. */
const MIN_CHUNK_CHARS = 60;
const MAX_CHUNK_CHARS = 400;
/**
* Split text into sentence-aligned chunks for pipelined synthesis.
* Sentences are merged until MIN_CHUNK_CHARS and hard-split at
* MAX_CHUNK_CHARS so a single run-on sentence cannot stall the pipeline.
*/
export function splitTextForSynthesis(text: string): string[] {
const normalized = text.replace(/\s+/g, ' ').trim();
if (!normalized) {
return [];
}
const sentences = normalized.match(/[^.!?…]+[.!?…]+["')\]]*\s*|[^.!?…]+$/g) ?? [normalized];
const chunks: string[] = [];
let current = '';
for (const sentence of sentences) {
for (let offset = 0; offset < sentence.length; offset += MAX_CHUNK_CHARS) {
const piece = sentence.slice(offset, offset + MAX_CHUNK_CHARS);
if (current && current.length + piece.length > MAX_CHUNK_CHARS) {
chunks.push(current.trim());
current = '';
}
current += piece;
if (current.length >= MIN_CHUNK_CHARS && /[.!?…]["')\]]*\s*$/.test(current)) {
chunks.push(current.trim());
current = '';
}
}
}
if (current.trim()) {
chunks.push(current.trim());
}
// Latency trim for the FIRST chunk only: if the opening sentence is long,
// split it at a clause boundary (comma/semicolon/colon) so the first
// audio starts sooner. Only the first chunk gets this treatment — comma
// splits inside every sentence would chop the prosody, and after playback
// starts the pipeline hides synthesis time anyway.
if (chunks.length > 0 && chunks[0].length > 120) {
const first = chunks[0];
const clauseBreak = /[,;:]\s/g;
let bestBreak = -1;
let match: RegExpExecArray | null;
while ((match = clauseBreak.exec(first)) !== null) {
const end = match.index + 1;
if (end < 40) {
continue;
}
if (end > 160) {
break;
}
bestBreak = end;
break;
}
if (bestBreak > 0) {
const head = first.slice(0, bestBreak).trim();
const tail = first.slice(bestBreak).trim();
chunks.splice(0, 1, head, tail);
}
}
return chunks;
}
let sharedAudioContext: AudioContext | null = null;
function getAudioContext(): AudioContext {
if (!sharedAudioContext) {
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
}
return sharedAudioContext;
}
interface PlaybackSession {
cancelled: boolean;
abort: AbortController;
}
export function useLocalTTS(): UseLocalTTSReturn {
const [isPlaying, setIsPlaying] = useState(false);
const [error, setError] = useState<string | null>(null);
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
const sessionRef = useRef<PlaybackSession | null>(null);
const unlockAudio = useCallback(async (): Promise<void> => {
try {
const ctx = getAudioContext();
if (ctx.state === 'suspended') {
await ctx.resume();
}
const buffer = ctx.createBuffer(1, 1, 22050);
const source = ctx.createBufferSource();
source.buffer = buffer;
source.connect(ctx.destination);
source.start(0);
} catch {
// Unlocking is best-effort.
}
}, []);
const stop = useCallback(() => {
const session = sessionRef.current;
if (session) {
session.cancelled = true;
session.abort.abort();
sessionRef.current = null;
}
if (audioSourceRef.current) {
try {
audioSourceRef.current.onended = null;
audioSourceRef.current.stop();
} catch {
// Already stopped
}
audioSourceRef.current = null;
}
setIsPlaying(false);
}, []);
const speak = useCallback(async (text: string, options?: LocalTTSSpeakOptions): Promise<void> => {
stop();
const chunks = splitTextForSynthesis(text);
if (chunks.length === 0) {
setError('No text to speak');
options?.onError?.('No text to speak');
return;
}
setError(null);
const session: PlaybackSession = { cancelled: false, abort: new AbortController() };
sessionRef.current = session;
const fetchChunk = async (chunk: string): Promise<ArrayBuffer> => {
const response = await runtimeFetch('/api/dictation/tts/speak', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
text: chunk,
...(typeof options?.speakerId === 'number' ? { speakerId: options.speakerId } : {}),
...(typeof options?.speed === 'number' ? { speed: options.speed } : {}),
}),
signal: session.abort.signal,
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
throw new Error(errorData.error || `HTTP ${response.status}`);
}
return (await response.blob()).arrayBuffer();
};
const playBuffer = async (arrayBuffer: ArrayBuffer): Promise<void> => {
const ctx = getAudioContext();
if (ctx.state === 'suspended') {
await ctx.resume();
}
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
if (session.cancelled) {
return;
}
await new Promise<void>((resolve) => {
const source = ctx.createBufferSource();
source.buffer = audioBuffer;
source.connect(ctx.destination);
audioSourceRef.current = source;
source.onended = () => {
if (audioSourceRef.current === source) {
audioSourceRef.current = null;
}
resolve();
};
source.start(0);
});
};
try {
setIsPlaying(true);
options?.onStart?.();
// Pipelined: synthesize chunk N+1 while chunk N is playing.
let nextFetch: Promise<ArrayBuffer> = fetchChunk(chunks[0]);
for (let i = 0; i < chunks.length; i += 1) {
const buffer = await nextFetch;
if (session.cancelled) {
return;
}
if (i + 1 < chunks.length) {
nextFetch = fetchChunk(chunks[i + 1]);
}
await playBuffer(buffer);
if (session.cancelled) {
return;
}
}
setIsPlaying(false);
options?.onEnd?.();
} catch (err) {
if ((err as Error).name === 'AbortError' || session.cancelled) {
return;
}
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
setError(errorMsg);
options?.onError?.(errorMsg);
setIsPlaying(false);
} finally {
if (sessionRef.current === session) {
sessionRef.current = null;
}
}
}, [stop]);
useEffect(() => {
return () => {
stop();
};
}, [stop]);
return {
isPlaying,
error,
speak,
stop,
unlockAudio,
};
}
+14 -1
View File
@@ -9,6 +9,7 @@ import { useCallback, useState } from 'react';
import { useConfigStore } from '@/stores/useConfigStore';
import { useServerTTS } from './useServerTTS';
import { useSayTTS } from './useSayTTS';
import { useLocalTTS } from './useLocalTTS';
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
import { sanitizeForTTS } from '@/lib/voice/summarize';
@@ -29,6 +30,7 @@ export function useMessageTTS(): UseMessageTTSReturn {
const speechPitch = useConfigStore((state) => state.speechPitch);
const speechVolume = useConfigStore((state) => state.speechVolume);
const sayVoice = useConfigStore((state) => state.sayVoice);
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
const browserVoice = useConfigStore((state) => state.browserVoice);
const openaiVoice = useConfigStore((state) => state.openaiVoice);
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
@@ -48,13 +50,15 @@ export function useMessageTTS(): UseMessageTTSReturn {
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS({
enabled: shouldCheckSayAvailability,
});
const { speak: speakLocalTTS, stop: stopLocalTTS } = useLocalTTS();
const stop = useCallback(() => {
setIsPlaying(false);
stopServerTTS();
stopSayTTS();
stopLocalTTS();
browserVoiceService.cancelSpeech();
}, [stopServerTTS, stopSayTTS]);
}, [stopServerTTS, stopSayTTS, stopLocalTTS]);
const play = useCallback(async (text: string) => {
if (!text.trim()) return;
@@ -84,6 +88,13 @@ export function useMessageTTS(): UseMessageTTSReturn {
onEnd: () => setIsPlaying(false),
onError: () => setIsPlaying(false),
});
} else if (voiceProvider === 'local') {
await speakLocalTTS(sanitizedText, {
speakerId: localTtsVoiceId,
speed: speechRate,
onEnd: () => setIsPlaying(false),
onError: () => setIsPlaying(false),
});
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
await speakSayTTS(sanitizedText, {
@@ -129,6 +140,8 @@ export function useMessageTTS(): UseMessageTTSReturn {
ttsInputMode,
speakServerTTS,
speakSayTTS,
speakLocalTTS,
localTtsVoiceId,
stop,
]);
-57
View File
@@ -1,57 +0,0 @@
import { useEffect, useRef } from 'react';
import { useSessionUIStore } from '@/sync/session-ui-store';
import { useSessionPermissions, useSessionTextMessages } from '@/sync/sync-context';
import { voiceHooks, isVoiceSessionStarted } from '@/lib/voice';
/**
* Hook that syncs session events (messages, permissions) to the voice agent.
* Call this inside VoiceProvider to enable session awareness during voice.
*/
export function useVoiceContext() {
const currentSessionId = useSessionUIStore((s) => s.currentSessionId);
const messages = useSessionTextMessages(currentSessionId ?? '');
const permissions = useSessionPermissions(currentSessionId ?? '');
// Track last seen message count to only forward new messages
const lastMessageCountRef = useRef(0);
// Forward new messages to voice agent
useEffect(() => {
if (!currentSessionId || !messages || messages.length === 0 || !isVoiceSessionStarted()) return;
const currentCount = messages.length;
if (currentCount <= lastMessageCountRef.current) return;
// Get only new messages (messages since last check)
const newMessages = messages.slice(lastMessageCountRef.current);
lastMessageCountRef.current = currentCount;
const formattedMessages = newMessages.map(m => ({
role: m.role ?? '',
content: m.text,
}));
voiceHooks.onMessages(currentSessionId, formattedMessages);
}, [currentSessionId, messages]);
// Forward permission requests to voice agent
useEffect(() => {
if (!currentSessionId || !permissions || permissions.length === 0) return;
if (!isVoiceSessionStarted()) return;
const request = permissions[0];
if (!request) return;
voiceHooks.onPermissionRequested(
currentSessionId,
request.id,
request.permission,
request.metadata
);
}, [currentSessionId, permissions]);
// Reset message count when session changes
useEffect(() => {
lastMessageCountRef.current = 0;
}, [currentSessionId]);
}
+3 -5
View File
@@ -178,14 +178,12 @@ export type DesktopSettings = {
responseStyleEnabled?: boolean;
responseStylePreset?: 'concise' | 'detailed' | 'mentor' | 'pushback' | 'noFiller' | 'matchEnergy' | 'warmPeer' | 'custom';
responseStyleCustomInstructions?: string;
sttProvider?: 'browser' | 'server' | 'wasm';
dictationEnabled?: boolean;
sttProvider?: 'local' | 'openai-compatible';
sttServerUrl?: string;
sttModel?: string;
wasmSttModel?: string;
sttLocalModel?: string;
sttLanguage?: string;
sttSilenceThresholdDb?: number;
sttSilenceHoldMs?: number;
sttTranscribeOnStop?: boolean;
// Global draft welcome starters (pinned commands/skills), persisted to settings.json
draftStarters?: DraftStarterRef[];
};
@@ -0,0 +1,447 @@
/**
* WebSocket client for the OpenChamber dictation endpoint (/api/dictation/ws).
*
* One shared client per app. The socket is opened lazily when a dictation
* starts and closed after an idle delay. URLs are resolved at connect time via
* the runtime URL resolver so runtime switches never leak a stale endpoint.
*/
import { getRuntimeUrlResolver } from '@/lib/runtime-url';
import { refreshRuntimeUrlAuthToken } from '@/lib/runtime-auth';
export interface DictationStartOptions {
provider?: 'local' | 'openai-compatible';
language?: string;
localModel?: string;
openaiCompatible?: {
baseUrl?: string;
model?: string;
apiKey?: string;
};
}
interface DictationServerMessage {
type: string;
dictationId?: string;
ackSeq?: number;
text?: string;
timeoutMs?: number;
error?: string;
retryable?: boolean;
reasonCode?: string;
}
interface DictationStreamError extends Error {
retryable: boolean;
reasonCode?: string;
}
const createStreamError = (message: string, retryable: boolean, reasonCode?: string): DictationStreamError => {
const error = new Error(message) as DictationStreamError;
error.name = 'DictationStreamError';
error.retryable = retryable;
if (reasonCode) {
error.reasonCode = reasonCode;
}
return error;
};
const CONNECT_TIMEOUT_MS = 10000;
const START_TIMEOUT_MS = 15000;
const IDLE_CLOSE_DELAY_MS = 30000;
const DEFAULT_FINISH_TIMEOUT_MS = 30000;
type ConnectionStatusListener = (connected: boolean) => void;
type PartialListener = (dictationId: string, text: string) => void;
interface PendingStart {
resolve: () => void;
reject: (error: Error) => void;
timeout: ReturnType<typeof setTimeout>;
}
interface PendingFinish {
resolve: (result: { text: string }) => void;
reject: (error: Error) => void;
timeout: ReturnType<typeof setTimeout> | null;
}
export class DictationClient {
private socket: WebSocket | null = null;
private connectPromise: Promise<void> | null = null;
private idleCloseTimer: ReturnType<typeof setTimeout> | null = null;
private readonly pendingStarts = new Map<string, PendingStart>();
private readonly pendingFinishes = new Map<string, PendingFinish>();
private readonly connectionListeners = new Set<ConnectionStatusListener>();
private readonly partialListeners = new Set<PartialListener>();
private activeDictations = 0;
get isConnected(): boolean {
return this.socket?.readyState === WebSocket.OPEN;
}
subscribeConnectionStatus(listener: ConnectionStatusListener): () => void {
this.connectionListeners.add(listener);
return () => {
this.connectionListeners.delete(listener);
};
}
onPartial(listener: PartialListener): () => void {
this.partialListeners.add(listener);
return () => {
this.partialListeners.delete(listener);
};
}
async ensureConnected(): Promise<void> {
if (this.isConnected) {
return;
}
if (this.connectPromise) {
await this.connectPromise;
return;
}
// A WebSocket upgrade can't carry an Authorization header, so it
// authenticates via the oc_url_token query param. Mint/await a valid
// token BEFORE connecting — the sync getter returns "" while the token
// is unminted or inside its expiry skew, and the server would reject
// the upgrade with 401.
try {
await refreshRuntimeUrlAuthToken();
} catch {
// No auth configured (local runtime) — proceed without a token.
}
this.connectPromise = new Promise<void>((resolve, reject) => {
let settled = false;
let socket: WebSocket;
try {
const url = getRuntimeUrlResolver().websocket('/api/dictation/ws');
socket = new WebSocket(url);
} catch (error) {
this.connectPromise = null;
reject(error instanceof Error ? error : new Error(String(error)));
return;
}
const timeout = setTimeout(() => {
if (!settled) {
settled = true;
this.connectPromise = null;
try {
socket.close();
} catch {
// ignore
}
reject(new Error('Dictation connection timed out'));
}
}, CONNECT_TIMEOUT_MS);
socket.onopen = () => {
// Wait for the server 'ready' frame before resolving.
};
socket.onmessage = (event) => {
let message: DictationServerMessage;
try {
message = JSON.parse(String(event.data));
} catch {
return;
}
if (!settled && message.type === 'ready') {
settled = true;
clearTimeout(timeout);
this.socket = socket;
this.connectPromise = null;
this.notifyConnection(true);
resolve();
return;
}
this.handleMessage(message);
};
socket.onerror = () => {
if (!settled) {
settled = true;
clearTimeout(timeout);
this.connectPromise = null;
reject(new Error('Dictation connection failed'));
}
};
socket.onclose = () => {
if (!settled) {
settled = true;
clearTimeout(timeout);
this.connectPromise = null;
reject(new Error('Dictation connection closed'));
return;
}
if (this.socket === socket) {
this.socket = null;
this.rejectAllPending(new Error('Dictation connection lost'));
this.notifyConnection(false);
}
};
});
await this.connectPromise;
}
/**
* Start a dictation stream. Resolves once the server acks the stream.
*/
async startDictationStream(
dictationId: string,
format: string,
options: DictationStartOptions,
): Promise<void> {
await this.ensureConnected();
this.activeDictations += 1;
this.clearIdleCloseTimer();
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
this.pendingStarts.delete(dictationId);
this.releaseDictation();
reject(new Error('Dictation start timed out'));
}, START_TIMEOUT_MS);
this.pendingStarts.set(dictationId, {
resolve: () => {
clearTimeout(timeout);
resolve();
},
reject: (error) => {
clearTimeout(timeout);
this.releaseDictation();
reject(error);
},
timeout });
if (!this.send({ type: 'start', dictationId, format, options })) {
clearTimeout(timeout);
this.pendingStarts.delete(dictationId);
this.releaseDictation();
reject(new Error('Dictation connection lost'));
}
});
}
sendDictationStreamChunk(dictationId: string, seq: number, audioBase64: string): boolean {
return this.send({ type: 'chunk', dictationId, seq, audio: audioBase64 });
}
/**
* Finish a dictation stream. Resolves with the final transcript.
*/
finishDictationStream(dictationId: string, finalSeq: number): Promise<{ text: string }> {
return new Promise<{ text: string }>((resolve, reject) => {
const pending: PendingFinish = {
resolve: (result) => {
if (pending.timeout) {
clearTimeout(pending.timeout);
}
this.releaseDictation();
resolve(result);
},
reject: (error) => {
if (pending.timeout) {
clearTimeout(pending.timeout);
}
this.releaseDictation();
reject(error);
},
timeout: null,
};
pending.timeout = setTimeout(() => {
this.pendingFinishes.delete(dictationId);
this.releaseDictation();
reject(new Error('Timed out waiting for transcription'));
}, DEFAULT_FINISH_TIMEOUT_MS);
this.pendingFinishes.set(dictationId, pending);
if (!this.send({ type: 'finish', dictationId, finalSeq })) {
this.pendingFinishes.delete(dictationId);
pending.reject(new Error('Dictation connection lost'));
}
});
}
cancelDictationStream(dictationId: string): void {
this.send({ type: 'cancel', dictationId });
const start = this.pendingStarts.get(dictationId);
if (start) {
this.pendingStarts.delete(dictationId);
start.reject(new Error('Dictation cancelled'));
}
const finish = this.pendingFinishes.get(dictationId);
if (finish) {
this.pendingFinishes.delete(dictationId);
finish.reject(new Error('Dictation cancelled'));
}
this.releaseDictation();
this.scheduleIdleCloseIfReady();
}
private handleMessage(message: DictationServerMessage): void {
const dictationId = message.dictationId;
if (!dictationId) {
return;
}
switch (message.type) {
case 'ack': {
const pendingStart = this.pendingStarts.get(dictationId);
if (pendingStart) {
this.pendingStarts.delete(dictationId);
pendingStart.resolve();
}
return;
}
case 'partial': {
for (const listener of this.partialListeners) {
listener(dictationId, message.text ?? '');
}
return;
}
case 'finish_accepted': {
const pendingFinish = this.pendingFinishes.get(dictationId);
if (pendingFinish && typeof message.timeoutMs === 'number') {
if (pendingFinish.timeout) {
clearTimeout(pendingFinish.timeout);
}
pendingFinish.timeout = setTimeout(() => {
this.pendingFinishes.delete(dictationId);
pendingFinish.reject(new Error('Timed out waiting for transcription'));
}, message.timeoutMs + 5000);
}
return;
}
case 'final': {
const pendingFinish = this.pendingFinishes.get(dictationId);
if (pendingFinish) {
this.pendingFinishes.delete(dictationId);
pendingFinish.resolve({ text: message.text ?? '' });
}
this.scheduleIdleCloseIfReady();
return;
}
case 'error': {
const error = createStreamError(
message.error || 'Dictation failed',
message.retryable !== false,
message.reasonCode,
);
const pendingStart = this.pendingStarts.get(dictationId);
if (pendingStart) {
this.pendingStarts.delete(dictationId);
pendingStart.reject(error);
}
const pendingFinish = this.pendingFinishes.get(dictationId);
if (pendingFinish) {
this.pendingFinishes.delete(dictationId);
pendingFinish.reject(error);
}
this.scheduleIdleCloseIfReady();
return;
}
default:
}
}
private send(message: object): boolean {
if (!this.isConnected || !this.socket) {
return false;
}
try {
this.socket.send(JSON.stringify(message));
return true;
} catch {
return false;
}
}
private notifyConnection(connected: boolean): void {
for (const listener of this.connectionListeners) {
listener(connected);
}
}
private rejectAllPending(error: Error): void {
for (const [dictationId, pending] of this.pendingStarts) {
this.pendingStarts.delete(dictationId);
pending.reject(error);
}
for (const [dictationId, pending] of this.pendingFinishes) {
this.pendingFinishes.delete(dictationId);
pending.reject(error);
}
this.activeDictations = 0;
}
private releaseDictation(): void {
this.activeDictations = Math.max(0, this.activeDictations - 1);
this.scheduleIdleCloseIfReady();
}
private scheduleIdleCloseIfReady(): void {
if (this.activeDictations > 0 || this.pendingFinishes.size > 0 || this.pendingStarts.size > 0) {
return;
}
this.clearIdleCloseTimer();
this.idleCloseTimer = setTimeout(() => {
if (this.activeDictations === 0 && this.pendingFinishes.size === 0 && this.pendingStarts.size === 0) {
const socket = this.socket;
this.socket = null;
if (socket) {
try {
socket.close(1000, 'idle');
} catch {
// ignore
}
this.notifyConnection(false);
}
}
}, IDLE_CLOSE_DELAY_MS);
}
/**
* Runtime switch: close the socket and fail all in-flight dictations so
* nothing keeps streaming to the previous runtime.
*/
cancelAllForRuntimeSwitch(): void {
this.clearIdleCloseTimer();
const socket = this.socket;
this.socket = null;
this.connectPromise = null;
this.rejectAllPending(new Error('Runtime changed'));
if (socket) {
try {
socket.close(1000, 'runtime switch');
} catch {
// ignore
}
this.notifyConnection(false);
}
}
private clearIdleCloseTimer(): void {
if (this.idleCloseTimer) {
clearTimeout(this.idleCloseTimer);
this.idleCloseTimer = null;
}
}
}
export const dictationClient = new DictationClient();
if (typeof window !== 'undefined') {
window.addEventListener('openchamber:runtime-endpoint-changed', () => {
// Drop the socket so the next dictation reconnects to the new runtime.
dictationClient.cancelAllForRuntimeSwitch();
});
}
@@ -0,0 +1,240 @@
/**
* Small, non-React state machine for dictation streaming.
*
* Responsibilities:
* - Maintain an ordered buffer of base64 PCM segments
* - Start/restart a dictation stream (dictationId)
* - Send missing segments (seq) when connected
* - Finish/cancel the stream
*
* Segments are retained until the dictation completes, which enables replay
* after a connection drop (`resetStreamForReplay()` + `finish()`).
*/
import type { DictationClient, DictationStartOptions } from './dictation-client';
const MAX_CHUNKS_PER_FLUSH_TURN = 128;
const PCM_DICTATION_FORMAT = 'audio/pcm;rate=16000;bits=16';
const waitForNextFlushTurn = (): Promise<void> => new Promise((resolve) => setTimeout(resolve, 0));
const createDictationIdDefault = (): string => {
const rand = Math.random().toString(36).slice(2, 10);
return `dic_${Date.now().toString(16)}${rand}`;
};
export interface DictationFinishResult {
dictationId: string;
text: string;
}
export class DictationStreamSender {
private readonly client: DictationClient;
private readonly format: string;
private readonly createDictationId: () => string;
private getStartOptions: () => DictationStartOptions;
private dictationId: string | null = null;
private sendSeq = 0;
private segments: string[] = [];
private streamReady = false;
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private drainWaiters: Array<() => void> = [];
private startGeneration = 0;
private startPromise: Promise<void> | null = null;
constructor(params: {
client: DictationClient;
getStartOptions: () => DictationStartOptions;
format?: string;
createDictationId?: () => string;
}) {
this.client = params.client;
this.format = params.format ?? PCM_DICTATION_FORMAT;
this.getStartOptions = params.getStartOptions;
this.createDictationId = params.createDictationId ?? createDictationIdDefault;
}
getDictationId(): string | null {
return this.dictationId;
}
getFinalSeq(): number {
return this.segments.length - 1;
}
hasSegments(): boolean {
return this.segments.length > 0;
}
clearAll(): void {
this.clearScheduledFlush();
this.dictationId = null;
this.sendSeq = 0;
this.segments = [];
this.streamReady = false;
this.startPromise = null;
this.startGeneration += 1;
}
resetStreamForReplay(): void {
this.clearScheduledFlush();
this.dictationId = null;
this.sendSeq = 0;
this.streamReady = false;
this.startPromise = null;
this.startGeneration += 1;
}
enqueueSegment(base64Pcm: string): void {
this.segments.push(base64Pcm);
if (!this.client.isConnected) {
return;
}
if (!this.dictationId) {
if (!this.startPromise) {
void this.restartStream().catch(() => {
// Start failures surface through finish(); segments are retained.
});
}
return;
}
this.flush();
}
flush(): number {
const dictationId = this.dictationId;
if (!this.client.isConnected || !dictationId || !this.streamReady) {
return 0;
}
let sent = 0;
while (this.sendSeq < this.segments.length && sent < MAX_CHUNKS_PER_FLUSH_TURN) {
const seq = this.sendSeq;
const audio = this.segments[seq];
if (!this.client.sendDictationStreamChunk(dictationId, seq, audio)) {
break;
}
this.sendSeq = seq + 1;
sent += 1;
}
if (this.hasPendingSegments()) {
this.scheduleFlush();
} else {
this.resolveDrainWaiters();
}
return sent;
}
async restartStream(): Promise<void> {
this.startGeneration += 1;
const generation = this.startGeneration;
const dictationId = this.createDictationId();
this.dictationId = dictationId;
this.sendSeq = 0;
this.streamReady = false;
const start = (async () => {
await this.client.startDictationStream(dictationId, this.format, this.getStartOptions());
if (this.startGeneration !== generation) {
return;
}
if (this.dictationId !== dictationId) {
return;
}
this.streamReady = true;
this.flush();
})()
.catch((error) => {
// Keep segments for retry, but clear the stream so finish can error cleanly.
if (this.startGeneration === generation && this.dictationId === dictationId) {
this.dictationId = null;
this.streamReady = false;
}
throw error;
})
.finally(() => {
if (this.startPromise === start) {
this.startPromise = null;
}
});
this.startPromise = start;
await start;
}
async finish(finalSeq: number): Promise<DictationFinishResult> {
if (!this.dictationId) {
await this.restartStream();
}
if (this.startPromise) {
await this.startPromise;
}
const dictationId = this.dictationId;
if (!dictationId || !this.streamReady) {
throw new Error('Failed to start dictation stream');
}
this.flush();
await this.waitForFlushDrain();
const result = await this.client.finishDictationStream(dictationId, finalSeq);
return { dictationId, text: result.text };
}
cancel(): void {
const dictationId = this.dictationId;
if (this.client.isConnected && dictationId) {
this.client.cancelDictationStream(dictationId);
}
this.resetStreamForReplay();
}
private hasPendingSegments(): boolean {
return this.sendSeq < this.segments.length;
}
private scheduleFlush(): void {
if (this.flushTimer) {
return;
}
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flush();
}, 0);
}
private clearScheduledFlush(): void {
if (!this.flushTimer) {
return;
}
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
private async waitForFlushDrain(): Promise<void> {
while (this.hasPendingSegments()) {
if (!this.client.isConnected || !this.dictationId || !this.streamReady) {
throw new Error('Failed to flush dictation stream');
}
await new Promise<void>((resolve) => {
this.drainWaiters.push(resolve);
});
await waitForNextFlushTurn();
}
}
private resolveDrainWaiters(): void {
const waiters = this.drainWaiters;
this.drainWaiters = [];
for (const resolve of waiters) {
resolve();
}
}
}
@@ -0,0 +1,301 @@
/**
* Microphone capture for dictation.
*
* Captures mono audio via getUserMedia, taps it with a ScriptProcessorNode
* (universally supported, including iOS WKWebView), resamples Float32 to
* 16 kHz PCM16LE, and emits ~1-second base64 chunks plus a normalized RMS
* volume for the level meter.
*/
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
export interface DictationAudioSourceConfig {
onPcmSegment: (base64Pcm: string) => void;
onError?: (error: Error) => void;
}
export interface DictationAudioSource {
start: () => Promise<void>;
stop: () => Promise<void>;
volume: number;
}
const OUTPUT_RATE = 16000;
const CHUNK_SAMPLES = OUTPUT_RATE; // ~1s per chunk
const getAudioContextCtor = (): typeof AudioContext | null => {
if (typeof window === 'undefined') {
return null;
}
const win = window as typeof window & { webkitAudioContext?: typeof AudioContext };
return win.AudioContext || win.webkitAudioContext || null;
};
const floatToInt16 = (sample: number): number => {
const clamped = Math.max(-1, Math.min(1, sample));
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
};
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
if (input.length === 0) {
return new Int16Array(0);
}
if (inputRate === outputRate) {
const out = new Int16Array(input.length);
for (let i = 0; i < input.length; i++) {
out[i] = floatToInt16(input[i]);
}
return out;
}
const ratio = inputRate / outputRate;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const out = new Int16Array(outputLength);
for (let i = 0; i < outputLength; i++) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const i1 = Math.min(input.length - 1, i0 + 1);
const frac = sourceIndex - i0;
out[i] = floatToInt16(input[i0] * (1 - frac) + input[i1] * frac);
}
return out;
};
const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => {
if (a.length === 0) {
return b;
}
if (b.length === 0) {
return a;
}
const out = new Int16Array(a.length + b.length);
out.set(a, 0);
out.set(b, a.length);
return out;
};
const int16ToBase64 = (pcm: Int16Array): string => {
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
let binary = '';
for (let i = 0; i < bytes.length; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
};
interface CaptureGraph {
stream: MediaStream | null;
context: AudioContext | null;
source: MediaStreamAudioSourceNode | null;
processor: ScriptProcessorNode | null;
gain: GainNode | null;
pending: Int16Array;
started: boolean;
}
const emptyGraph = (): CaptureGraph => ({
stream: null,
context: null,
source: null,
processor: null,
gain: null,
pending: new Int16Array(0),
started: false,
});
const safeDisconnect = (node: AudioNode | null): void => {
if (!node) {
return;
}
try {
node.disconnect();
} catch {
// no-op
}
};
export const isDictationCaptureSupported = (): boolean => {
if (typeof navigator === 'undefined' || typeof window === 'undefined') {
return false;
}
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
return false;
}
return getAudioContextCtor() !== null;
};
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
const [volume, setVolume] = useState(0);
const onPcmSegmentRef = useRef(config.onPcmSegment);
const onErrorRef = useRef(config.onError);
useEffect(() => {
onPcmSegmentRef.current = config.onPcmSegment;
onErrorRef.current = config.onError;
}, [config.onPcmSegment, config.onError]);
const graphRef = useRef<CaptureGraph>(emptyGraph());
const start = useCallback(async () => {
if (graphRef.current.started) {
return;
}
if (
typeof navigator === 'undefined' ||
!navigator.mediaDevices ||
typeof navigator.mediaDevices.getUserMedia !== 'function'
) {
throw new Error('Microphone capture is not supported in this environment');
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error('AudioContext unavailable');
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
noiseSuppression: true,
echoCancellation: true,
autoGainControl: true,
},
});
const context = new AudioContextCtor();
try {
if (context.state === 'suspended') {
await context.resume().catch(() => undefined);
}
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(4096, 1, 1);
const gain = context.createGain();
gain.gain.value = 0;
graphRef.current = {
stream,
context,
source,
processor,
gain,
pending: new Int16Array(0),
started: true,
};
processor.onaudioprocess = (event) => {
const graph = graphRef.current;
if (!graph.started) {
return;
}
const input = event.inputBuffer.getChannelData(0);
let sumSquares = 0;
for (let i = 0; i < input.length; i++) {
sumSquares += input[i] * input[i];
}
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
setVolume(Math.min(1, Math.max(0, rms * 2)));
const next = resampleToPcm16(input, context.sampleRate, OUTPUT_RATE);
graph.pending = concatInt16(graph.pending, next);
while (graph.pending.length >= CHUNK_SAMPLES) {
const chunk = graph.pending.slice(0, CHUNK_SAMPLES);
graph.pending = graph.pending.slice(CHUNK_SAMPLES);
onPcmSegmentRef.current(int16ToBase64(chunk));
}
};
source.connect(processor);
processor.connect(gain);
gain.connect(context.destination);
} catch (error) {
stream.getTracks().forEach((track) => {
try {
track.stop();
} catch {
// no-op
}
});
try {
await context.close();
} catch {
// no-op
}
graphRef.current = emptyGraph();
throw error instanceof Error ? error : new Error(String(error));
}
}, []);
const stop = useCallback(async () => {
const graph = graphRef.current;
graph.started = false;
setVolume(0);
if (graph.processor) {
try {
graph.processor.onaudioprocess = null;
} catch {
// no-op
}
}
safeDisconnect(graph.processor);
safeDisconnect(graph.source);
safeDisconnect(graph.gain);
if (graph.stream) {
graph.stream.getTracks().forEach((track) => {
try {
track.stop();
} catch {
// no-op
}
});
}
const pending = graph.pending;
graph.pending = new Int16Array(0);
if (pending.length > 0) {
onPcmSegmentRef.current(int16ToBase64(pending));
}
if (graph.context) {
try {
await graph.context.close();
} catch {
// no-op
}
}
// A new capture may have started while the old context was closing;
// only clear the ref if it still points at the graph we tore down.
if (graphRef.current === graph) {
graphRef.current = emptyGraph();
}
}, []);
useEffect(() => {
return () => {
void stop().catch((err) => {
onErrorRef.current?.(err instanceof Error ? err : new Error(String(err)));
});
};
}, [stop]);
return useMemo(
() => ({
start: async () => {
try {
await start();
} catch (err) {
const normalized = err instanceof Error ? err : new Error(String(err));
onErrorRef.current?.(normalized);
throw normalized;
}
},
stop,
volume,
}),
[start, stop, volume],
);
}
@@ -1518,16 +1518,33 @@ export const settingsDict = {
'settings.notifications.page.toast.backgroundDisabled': 'Background notifications disabled',
'settings.notifications.page.testNotification.title': 'Test Notification',
'settings.notifications.page.testNotification.body': 'This is a test notification from OpenChamber.',
'settings.voice.page.section.voiceSetup': 'Voice Setup',
'settings.voice.page.section.speechRecognition': 'Speech Recognition',
'settings.voice.page.field.enableVoiceInput': 'Enable voice input',
'settings.voice.page.field.enableVoiceInputAria': 'Enable voice input (dictation)',
'settings.voice.page.section.playbackAndSummary': 'Playback',
'settings.voice.page.field.enableVoiceModeAria': 'Enable voice mode',
'settings.voice.page.field.enableVoiceMode': 'Enable Voice Mode',
'settings.voice.page.field.provider': 'Provider',
'settings.voice.page.provider.browser': 'Browser',
'settings.voice.page.provider.custom': 'Custom',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': 'Server',
'settings.voice.page.provider.local': 'Local',
'settings.voice.page.tooltip.sttLocal': 'On-device transcription on the OpenChamber server. Models download automatically; no API key needed.',
'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro, English). The model downloads automatically; no API key needed.',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (English)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 European languages)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingual)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (multilingual)',
'settings.voice.page.stt.badge.bestForEnglish': 'Best for English',
'settings.voice.page.stt.badge.bestForMultilingual': 'Best for multilingual',
'settings.voice.page.stt.meta.accuracy': 'Accuracy',
'settings.voice.page.stt.meta.speed': 'Speed',
'settings.voice.page.stt.modelInstalled': 'Model installed',
'settings.voice.page.stt.modelDownloading': 'Downloading model...',
'settings.voice.page.stt.modelDownloadingProgress': 'Downloading model... {percent}%',
'settings.voice.page.stt.modelNotInstalled': 'Model not downloaded',
'settings.voice.page.stt.modelDownload': 'Download',
'settings.voice.page.stt.modelDelete': 'Delete model',
'settings.voice.page.stt.modelRetry': 'Retry',
'settings.voice.page.provider.wasm': 'Local',
'settings.voice.page.stt.wasmModel': 'Whisper Model',
'settings.voice.page.stt.wasmLoaded': 'Model loaded, ready',
+12
View File
@@ -1873,6 +1873,18 @@ export const dict = {
'chat.chatInput.placeholder.chat': '@ for files/agents; / for commands and skills; ! for shell; # for snippets',
'chat.chatInput.placeholder.chatCompact': 'Use @ / ! # for helpers',
'chat.chatInput.placeholder.selectSession': 'Select or create a session to start chatting',
'chat.dictation.start': 'Start dictation',
'chat.dictation.overlayAria': 'Dictation',
'chat.dictation.downloadingModel': 'Downloading speech model...',
'chat.dictation.downloadingModelProgress': 'Downloading speech model... {percent}%',
'chat.dictation.listening': 'Listening...',
'chat.dictation.processing': 'Transcribing...',
'chat.dictation.failed': 'Transcription failed',
'chat.dictation.cancel': 'Discard dictation',
'chat.dictation.insert': 'Insert transcript',
'chat.dictation.insertAndSend': 'Insert and send',
'chat.dictation.retry': 'Retry transcription',
'chat.dictation.discard': 'Discard recording',
'chat.snippetAutocomplete.action.addNew': '+ Add new snippet',
'chat.snippetAutocomplete.empty': 'No snippets found',
'chat.snippetAutocomplete.footer': '↑↓ navigate • Enter select • Esc close',
@@ -1485,16 +1485,33 @@ export const settingsDict = {
"settings.notifications.page.toast.backgroundDisabled": "Notificaciones de fondo deshabilitadas",
"settings.notifications.page.testNotification.title": "Notificación de prueba",
"settings.notifications.page.testNotification.body": "Esta es una notificación de prueba de OpenChamber.",
"settings.voice.page.section.voiceSetup": "Configuración de voz",
"settings.voice.page.section.speechRecognition": "Reconocimiento de voz",
"settings.voice.page.field.enableVoiceInput": "Habilitar entrada de voz",
"settings.voice.page.field.enableVoiceInputAria": "Habilitar entrada de voz (dictado)",
"settings.voice.page.section.playbackAndSummary": "Reproducción",
"settings.voice.page.field.enableVoiceModeAria": "Habilitar modo de voz",
"settings.voice.page.field.enableVoiceMode": "Habilitar modo de voz",
"settings.voice.page.field.provider": "Proveedor",
"settings.voice.page.provider.browser": "Navegador",
"settings.voice.page.provider.custom": "Personalizado",
"settings.voice.page.provider.say": "Decir",
"settings.voice.page.provider.server": "Servidor",
"settings.voice.page.provider.local": "Local",
"settings.voice.page.tooltip.sttLocal": "Transcripción local en el servidor de OpenChamber. Los modelos se descargan automáticamente; no se necesita clave de API.",
"settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro, inglés). El modelo se descarga automáticamente; no se necesita clave de API.",
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglés)",
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeos)",
"settings.voice.page.stt.model.whisperBase": "Whisper base (multilingüe)",
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (multilingüe)",
"settings.voice.page.stt.badge.bestForEnglish": "Mejor para inglés",
"settings.voice.page.stt.badge.bestForMultilingual": "Mejor multilingüe",
"settings.voice.page.stt.meta.accuracy": "Precisión",
"settings.voice.page.stt.meta.speed": "Velocidad",
"settings.voice.page.stt.modelInstalled": "Modelo instalado",
"settings.voice.page.stt.modelDownloading": "Descargando modelo...",
"settings.voice.page.stt.modelDownloadingProgress": "Descargando modelo... {percent}%",
"settings.voice.page.stt.modelNotInstalled": "Modelo no descargado",
"settings.voice.page.stt.modelDownload": "Descargar",
"settings.voice.page.stt.modelDelete": "Eliminar modelo",
"settings.voice.page.stt.modelRetry": "Reintentar",
"settings.voice.page.provider.wasm": "Local",
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
"settings.voice.page.stt.wasmLoaded": "Modelo cargado",
+12
View File
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
'diffView.reviewDialog.actions.starting': 'Iniciando...',
'diffView.reviewDialog.toast.noSessionDirectory': 'El directorio de la sesión no está disponible',
'diffView.reviewDialog.toast.startFailed': 'No se pudo iniciar el flujo de revisión',
'chat.dictation.start': 'Iniciar dictado',
'chat.dictation.overlayAria': 'Dictado',
'chat.dictation.downloadingModel': 'Descargando el modelo de voz...',
'chat.dictation.downloadingModelProgress': 'Descargando el modelo de voz... {percent}%',
'chat.dictation.listening': 'Escuchando...',
'chat.dictation.processing': 'Transcribiendo...',
'chat.dictation.failed': 'La transcripción falló',
'chat.dictation.cancel': 'Descartar dictado',
'chat.dictation.insert': 'Insertar transcripción',
'chat.dictation.insertAndSend': 'Insertar y enviar',
'chat.dictation.retry': 'Reintentar transcripción',
'chat.dictation.discard': 'Descartar grabación',
'chat.history.loadOlder': 'Cargar mensajes anteriores',
'chat.autoReview.title': 'El ciclo de revisión de código está en curso',
'chat.autoReview.status.waitingForReviewer': 'Esperando al revisor',
@@ -1468,16 +1468,33 @@ export const settingsDict = {
'settings.notifications.page.toast.backgroundDisabled': 'Notifications en arrière-plan désactivées',
'settings.notifications.page.testNotification.title': 'Avis de test',
'settings.notifications.page.testNotification.body': 'Il s\'agit d\'une notification de test de OpenChamber.',
'settings.voice.page.section.voiceSetup': 'Configuration vocale',
'settings.voice.page.section.speechRecognition': 'Reconnaissance vocale',
'settings.voice.page.field.enableVoiceInput': 'Activer la saisie vocale',
'settings.voice.page.field.enableVoiceInputAria': 'Activer la saisie vocale (dictée)',
'settings.voice.page.section.playbackAndSummary': 'Lecture',
'settings.voice.page.field.enableVoiceModeAria': 'Activer le mode vocal',
'settings.voice.page.field.enableVoiceMode': 'Activer le mode vocal',
'settings.voice.page.field.provider': 'Fournisseur',
'settings.voice.page.provider.browser': 'Navigateur',
'settings.voice.page.provider.custom': 'Personnalisé',
'settings.voice.page.provider.say': 'Dire',
'settings.voice.page.provider.server': 'Serveur',
'settings.voice.page.provider.local': 'Local',
'settings.voice.page.tooltip.sttLocal': 'Transcription locale sur le serveur OpenChamber. Les modèles se téléchargent automatiquement ; aucune clé d\'API requise.',
'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro, anglais). Le modèle se télécharge automatiquement ; aucune clé dAPI requise.',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (anglais)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 langues européennes)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingue)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (multilingue)',
'settings.voice.page.stt.badge.bestForEnglish': 'Idéal pour langlais',
'settings.voice.page.stt.badge.bestForMultilingual': 'Idéal multilingue',
'settings.voice.page.stt.meta.accuracy': 'Précision',
'settings.voice.page.stt.meta.speed': 'Vitesse',
'settings.voice.page.stt.modelInstalled': 'Modèle installé',
'settings.voice.page.stt.modelDownloading': 'Téléchargement du modèle...',
'settings.voice.page.stt.modelDownloadingProgress': 'Téléchargement du modèle... {percent}%',
'settings.voice.page.stt.modelNotInstalled': 'Modèle non téléchargé',
'settings.voice.page.stt.modelDownload': 'Télécharger',
'settings.voice.page.stt.modelDelete': 'Supprimer le modèle',
'settings.voice.page.stt.modelRetry': 'Réessayer',
'settings.voice.page.provider.wasm': 'Local',
'settings.voice.page.stt.wasmModel': 'Modèle Whisper',
'settings.voice.page.stt.wasmLoaded': 'Modèle chargé, prêt',
+12
View File
@@ -1669,6 +1669,18 @@ export const dict = {
'chat.chatInput.placeholder.chat': '@ pour les fichiers/agents ; / pour les commandes et les skills ; ! pour shell ; # pour les extraits',
'chat.chatInput.placeholder.chatCompact': 'Utiliser @ / ! # pour les aides',
'chat.chatInput.placeholder.selectSession': 'Sélectionnez ou créez une session pour commencer à discuter',
'chat.dictation.start': 'Démarrer la dictée',
'chat.dictation.overlayAria': 'Dictée',
'chat.dictation.downloadingModel': 'Téléchargement du modèle vocal...',
'chat.dictation.downloadingModelProgress': 'Téléchargement du modèle vocal... {percent}%',
'chat.dictation.listening': 'Écoute...',
'chat.dictation.processing': 'Transcription...',
'chat.dictation.failed': 'Échec de la transcription',
'chat.dictation.cancel': 'Abandonner la dictée',
'chat.dictation.insert': 'Insérer la transcription',
'chat.dictation.insertAndSend': 'Insérer et envoyer',
'chat.dictation.retry': 'Réessayer la transcription',
'chat.dictation.discard': 'Abandonner l\'enregistrement',
'chat.snippetAutocomplete.action.addNew': '+ Ajouter un nouvel extrait',
'chat.snippetAutocomplete.empty': 'Aucun extrait trouvé',
'chat.snippetAutocomplete.footer': '↑↓ naviguer • Entrer sélectionner • Esc fermer',
@@ -1518,16 +1518,33 @@ export const settingsDict = {
'settings.notifications.page.toast.backgroundDisabled': 'バックグラウンド通知を無効化しました',
'settings.notifications.page.testNotification.title': 'テスト通知',
'settings.notifications.page.testNotification.body': 'これは OpenChamber からのテスト通知です。',
'settings.voice.page.section.voiceSetup': '音声設定',
'settings.voice.page.section.speechRecognition': '音声認識',
'settings.voice.page.field.enableVoiceInput': '音声入力を有効にする',
'settings.voice.page.field.enableVoiceInputAria': '音声入力(ディクテーション)を有効にする',
'settings.voice.page.section.playbackAndSummary': '再生',
'settings.voice.page.field.enableVoiceModeAria': '音声モードを有効化',
'settings.voice.page.field.enableVoiceMode': '音声モードを有効化',
'settings.voice.page.field.provider': 'プロバイダー',
'settings.voice.page.provider.browser': 'ブラウザ',
'settings.voice.page.provider.custom': 'カスタム',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': 'サーバー',
'settings.voice.page.provider.local': 'ローカル',
'settings.voice.page.tooltip.sttLocal': 'OpenChamber サーバー上でローカルに文字起こしします。モデルは自動でダウンロードされ、API キーは不要です。',
'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します(Kokoro、英語)。モデルは自動でダウンロードされ、API キーは不要です。',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英語)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(ヨーロッパ25言語)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多言語)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多言語)',
'settings.voice.page.stt.badge.bestForEnglish': '英語に最適',
'settings.voice.page.stt.badge.bestForMultilingual': '多言語に最適',
'settings.voice.page.stt.meta.accuracy': '精度',
'settings.voice.page.stt.meta.speed': '速度',
'settings.voice.page.stt.modelInstalled': 'モデルはインストール済み',
'settings.voice.page.stt.modelDownloading': 'モデルをダウンロード中...',
'settings.voice.page.stt.modelDownloadingProgress': 'モデルをダウンロード中... {percent}%',
'settings.voice.page.stt.modelNotInstalled': 'モデル未ダウンロード',
'settings.voice.page.stt.modelDownload': 'ダウンロード',
'settings.voice.page.stt.modelDelete': 'モデルを削除',
'settings.voice.page.stt.modelRetry': '再試行',
'settings.voice.page.provider.wasm': 'ローカル',
'settings.voice.page.stt.wasmModel': 'Whisper モデル',
'settings.voice.page.stt.wasmLoaded': 'モデル読み込み完了',
+12
View File
@@ -1869,6 +1869,18 @@ export const dict: Record<I18nKey, string> = {
'chat.chatInput.placeholder.chat': '@でファイル/エージェント、/でコマンド/スキル、!でシェル、#でスニペット',
'chat.chatInput.placeholder.chatCompact': '@ / ! # でヘルパーを使用',
'chat.chatInput.placeholder.selectSession': 'セッションを選択または作成してチャットを開始',
'chat.dictation.start': '音声入力を開始',
'chat.dictation.overlayAria': '音声入力',
'chat.dictation.downloadingModel': '音声モデルをダウンロード中...',
'chat.dictation.downloadingModelProgress': '音声モデルをダウンロード中... {percent}%',
'chat.dictation.listening': '聞き取り中...',
'chat.dictation.processing': '文字起こし中...',
'chat.dictation.failed': '文字起こしに失敗しました',
'chat.dictation.cancel': '音声入力を破棄',
'chat.dictation.insert': 'テキストを挿入',
'chat.dictation.insertAndSend': '挿入して送信',
'chat.dictation.retry': '文字起こしを再試行',
'chat.dictation.discard': '録音を破棄',
'chat.snippetAutocomplete.action.addNew': '+ 新しいスニペットを追加',
'chat.snippetAutocomplete.empty': 'スニペットが見つかりません',
'chat.snippetAutocomplete.footer': '↑↓ 移動 • Enter 選択 • Esc 閉じる',
@@ -1485,16 +1485,33 @@ export const settingsDict = {
'settings.notifications.page.toast.backgroundDisabled': '백그라운드 알림이 비활성화되었습니다',
'settings.notifications.page.testNotification.title': '테스트 알림',
'settings.notifications.page.testNotification.body': 'OpenChamber의 테스트 알림입니다.',
'settings.voice.page.section.voiceSetup': '음성',
'settings.voice.page.section.speechRecognition': '음성 인식',
'settings.voice.page.field.enableVoiceInput': '음성 입력 사용',
'settings.voice.page.field.enableVoiceInputAria': '음성 입력(받아쓰기) 사용',
'settings.voice.page.section.playbackAndSummary': '재생',
'settings.voice.page.field.enableVoiceModeAria': '음성 모드 활성화',
'settings.voice.page.field.enableVoiceMode': '음성 모드 활성화',
'settings.voice.page.field.provider': '프로바이더',
'settings.voice.page.provider.browser': '브라우저',
'settings.voice.page.provider.custom': '사용자 정의',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': '서버',
'settings.voice.page.provider.local': '로컬',
'settings.voice.page.tooltip.sttLocal': 'OpenChamber 서버에서 로컬로 변환합니다. 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.',
'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(Kokoro, 영어). 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (영어)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (유럽 25개 언어)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base (다국어)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (다국어)',
'settings.voice.page.stt.badge.bestForEnglish': '영어에 최적',
'settings.voice.page.stt.badge.bestForMultilingual': '다국어에 최적',
'settings.voice.page.stt.meta.accuracy': '정확도',
'settings.voice.page.stt.meta.speed': '속도',
'settings.voice.page.stt.modelInstalled': '모델 설치됨',
'settings.voice.page.stt.modelDownloading': '모델 다운로드 중...',
'settings.voice.page.stt.modelDownloadingProgress': '모델 다운로드 중... {percent}%',
'settings.voice.page.stt.modelNotInstalled': '모델이 다운로드되지 않음',
'settings.voice.page.stt.modelDownload': '다운로드',
'settings.voice.page.stt.modelDelete': '모델 삭제',
'settings.voice.page.stt.modelRetry': '다시 시도',
'settings.voice.page.provider.wasm': '로컬',
'settings.voice.page.stt.wasmModel': 'Whisper 모델',
'settings.voice.page.stt.wasmLoaded': '모델 로드됨',
+12
View File
@@ -1873,6 +1873,18 @@ export const dict: Record<I18nKey, string> = {
'chat.chatInput.placeholder.chat': '@ 파일/에이전트; / 명령 및 스킬; ! shell; # 스니펫',
'chat.chatInput.placeholder.chatCompact': '@ / ! # 도우미 사용',
'chat.chatInput.placeholder.selectSession': '채팅을 시작할 세션을 선택하거나 새로 만드세요',
'chat.dictation.start': '받아쓰기 시작',
'chat.dictation.overlayAria': '받아쓰기',
'chat.dictation.downloadingModel': '음성 모델 다운로드 중...',
'chat.dictation.downloadingModelProgress': '음성 모델 다운로드 중... {percent}%',
'chat.dictation.listening': '듣는 중...',
'chat.dictation.processing': '변환 중...',
'chat.dictation.failed': '변환에 실패했습니다',
'chat.dictation.cancel': '받아쓰기 취소',
'chat.dictation.insert': '텍스트 삽입',
'chat.dictation.insertAndSend': '삽입 후 전송',
'chat.dictation.retry': '변환 다시 시도',
'chat.dictation.discard': '녹음 삭제',
'chat.snippetAutocomplete.action.addNew': '+ 새 스니펫 추가',
'chat.snippetAutocomplete.empty': '스니펫을 찾을 수 없음',
'chat.snippetAutocomplete.footer': '↑↓ 이동 • Enter 선택 • Esc 닫기',
@@ -1743,8 +1743,6 @@ export const settingsDict = {
'settings.voice.page.field.apiKeyHintUsingConfig': 'Użyto klucza z konfiguracji',
'settings.voice.page.field.auto': 'Auto',
'settings.voice.page.field.configuredAbove': 'Skonfigurowano powyżej',
'settings.voice.page.field.enableVoiceMode': 'Włącz tryb głosowy',
'settings.voice.page.field.enableVoiceModeAria': 'Włącz tryb głosowy',
'settings.voice.page.field.language': 'Język',
'settings.voice.page.field.messageReadAloudButton': 'Przycisk czytania wiadomości na głos',
'settings.voice.page.field.messageReadAloudButtonAria': 'Przycisk czytania wiadomości na głos',
@@ -1782,6 +1780,24 @@ export const settingsDict = {
'settings.voice.page.provider.custom': 'Własny',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': 'Serwer',
'settings.voice.page.provider.local': 'Lokalny',
'settings.voice.page.tooltip.sttLocal': 'Transkrypcja lokalna na serwerze OpenChamber. Modele pobierają się automatycznie; klucz API nie jest potrzebny.',
'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro, angielski). Model pobiera się automatycznie; klucz API nie jest potrzebny.',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (angielski)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 języków europejskich)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base (wielojęzyczny)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (wielojęzyczny)',
'settings.voice.page.stt.badge.bestForEnglish': 'Najlepszy do angielskiego',
'settings.voice.page.stt.badge.bestForMultilingual': 'Najlepszy wielojęzyczny',
'settings.voice.page.stt.meta.accuracy': 'Dokładność',
'settings.voice.page.stt.meta.speed': 'Szybkość',
'settings.voice.page.stt.modelInstalled': 'Model zainstalowany',
'settings.voice.page.stt.modelDownloading': 'Pobieranie modelu...',
'settings.voice.page.stt.modelDownloadingProgress': 'Pobieranie modelu... {percent}%',
'settings.voice.page.stt.modelNotInstalled': 'Model niepobrany',
'settings.voice.page.stt.modelDownload': 'Pobierz',
'settings.voice.page.stt.modelDelete': 'Usuń model',
'settings.voice.page.stt.modelRetry': 'Ponów',
'settings.voice.page.provider.wasm': 'Lokalny',
'settings.voice.page.stt.wasmModel': 'Model Whisper',
'settings.voice.page.stt.wasmLoaded': 'Model załadowany',
@@ -1792,7 +1808,8 @@ export const settingsDict = {
'settings.voice.page.stt.wasmRetry': 'Spróbuj ponownie',
'settings.voice.page.section.playbackAndSummary': 'Odtwarzanie',
'settings.voice.page.section.speechRecognition': 'Rozpoznawanie mowy',
'settings.voice.page.section.voiceSetup': 'Konfiguracja głosu',
'settings.voice.page.field.enableVoiceInput': 'Włącz wprowadzanie głosowe',
'settings.voice.page.field.enableVoiceInputAria': 'Włącz wprowadzanie głosowe (dyktowanie)',
'settings.voice.page.tooltip.browser': 'Darmowe, offline, ograniczone wsparcie mobilne.',
'settings.voice.page.tooltip.custom': 'Serwer zgodny z OpenAI (na przykład Kokoro).',
'settings.voice.page.tooltip.openai': 'Wysoka jakość, gotowe na urządzenia mobilne, wymaga klucza API.',
+12
View File
@@ -1091,6 +1091,18 @@ export const dict: Record<I18nKey, string> = {
'chat.chatInput.placeholder.chatCompact': 'Użyj @ / ! # dla pomocników',
'chat.chatInput.placeholder.selectSession': 'Wybierz lub utwórz sesję, aby zacząć czatować',
'chat.chatInput.placeholder.shell': 'Wpisz polecenie powłoki...',
'chat.dictation.start': 'Rozpocznij dyktowanie',
'chat.dictation.overlayAria': 'Dyktowanie',
'chat.dictation.downloadingModel': 'Pobieranie modelu mowy...',
'chat.dictation.downloadingModelProgress': 'Pobieranie modelu mowy... {percent}%',
'chat.dictation.listening': 'Słucham...',
'chat.dictation.processing': 'Transkrybowanie...',
'chat.dictation.failed': 'Transkrypcja nie powiodła się',
'chat.dictation.cancel': 'Odrzuć dyktowanie',
'chat.dictation.insert': 'Wstaw transkrypcję',
'chat.dictation.insertAndSend': 'Wstaw i wyślij',
'chat.dictation.retry': 'Ponów transkrypcję',
'chat.dictation.discard': 'Odrzuć nagranie',
'chat.snippetAutocomplete.action.addNew': '+ Dodaj nowy fragment',
'chat.snippetAutocomplete.empty': 'Nie znaleziono fragmentów',
'chat.snippetAutocomplete.footer': '↑↓ nawigacja • Enter wybierz • Esc zamknij',
@@ -1485,16 +1485,33 @@ export const settingsDict = {
"settings.notifications.page.toast.backgroundDisabled": "Notificações em segundo plano desabilitadas",
"settings.notifications.page.testNotification.title": "Notificação de teste",
"settings.notifications.page.testNotification.body": "Esta é uma notificação de teste do OpenChamber.",
"settings.voice.page.section.voiceSetup": "Configurações de voz",
"settings.voice.page.section.speechRecognition": "Reconhecimento de voz",
"settings.voice.page.field.enableVoiceInput": "Ativar entrada por voz",
"settings.voice.page.field.enableVoiceInputAria": "Ativar entrada por voz (ditado)",
"settings.voice.page.section.playbackAndSummary": "Reprodução",
"settings.voice.page.field.enableVoiceModeAria": "Ativar modo de voz",
"settings.voice.page.field.enableVoiceMode": "Ativar modo de voz",
"settings.voice.page.field.provider": "Provedor",
"settings.voice.page.provider.browser": "Navegador",
"settings.voice.page.provider.custom": "Personalizado",
"settings.voice.page.provider.say": "Falar",
"settings.voice.page.provider.server": "Servidor",
"settings.voice.page.provider.local": "Local",
"settings.voice.page.tooltip.sttLocal": "Transcrição local no servidor do OpenChamber. Os modelos são baixados automaticamente; não é necessária chave de API.",
"settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro, inglês). O modelo é baixado automaticamente; não é necessária chave de API.",
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglês)",
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeus)",
"settings.voice.page.stt.model.whisperBase": "Whisper base (multilíngue)",
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (multilíngue)",
"settings.voice.page.stt.badge.bestForEnglish": "Melhor para inglês",
"settings.voice.page.stt.badge.bestForMultilingual": "Melhor multilíngue",
"settings.voice.page.stt.meta.accuracy": "Precisão",
"settings.voice.page.stt.meta.speed": "Velocidade",
"settings.voice.page.stt.modelInstalled": "Modelo instalado",
"settings.voice.page.stt.modelDownloading": "Baixando modelo...",
"settings.voice.page.stt.modelDownloadingProgress": "Baixando modelo... {percent}%",
"settings.voice.page.stt.modelNotInstalled": "Modelo não baixado",
"settings.voice.page.stt.modelDownload": "Baixar",
"settings.voice.page.stt.modelDelete": "Excluir modelo",
"settings.voice.page.stt.modelRetry": "Tentar novamente",
"settings.voice.page.provider.wasm": "Local",
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
"settings.voice.page.stt.wasmLoaded": "Modelo carregado",
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
'diffView.reviewDialog.actions.starting': 'Iniciando...',
'diffView.reviewDialog.toast.noSessionDirectory': 'O diretório da sessão está indisponível',
'diffView.reviewDialog.toast.startFailed': 'Falha ao iniciar o fluxo de revisão',
'chat.dictation.start': 'Iniciar ditado',
'chat.dictation.overlayAria': 'Ditado',
'chat.dictation.downloadingModel': 'Baixando o modelo de fala...',
'chat.dictation.downloadingModelProgress': 'Baixando o modelo de fala... {percent}%',
'chat.dictation.listening': 'Ouvindo...',
'chat.dictation.processing': 'Transcrevendo...',
'chat.dictation.failed': 'Falha na transcrição',
'chat.dictation.cancel': 'Descartar ditado',
'chat.dictation.insert': 'Inserir transcrição',
'chat.dictation.insertAndSend': 'Inserir e enviar',
'chat.dictation.retry': 'Tentar transcrever novamente',
'chat.dictation.discard': 'Descartar gravação',
'chat.history.loadOlder': 'Carregar mensagens anteriores',
'chat.autoReview.title': 'O ciclo de revisão de código está em andamento',
'chat.autoReview.status.waitingForReviewer': 'Aguardando o revisor',
@@ -1485,16 +1485,33 @@ export const settingsDict = {
"settings.notifications.page.toast.backgroundDisabled": "Фонові сповіщення вимкнено",
"settings.notifications.page.testNotification.title": "Тестове сповіщення",
"settings.notifications.page.testNotification.body": "Це тестове сповіщення від OpenChamber.",
"settings.voice.page.section.voiceSetup": "Налаштування голосу",
"settings.voice.page.section.speechRecognition": "Розпізнавання мовлення",
"settings.voice.page.field.enableVoiceInput": "Увімкнути голосовий ввід",
"settings.voice.page.field.enableVoiceInputAria": "Увімкнути голосовий ввід (диктування)",
"settings.voice.page.section.playbackAndSummary": "Відтворення",
"settings.voice.page.field.enableVoiceModeAria": "Увімкнути голосовий режим",
"settings.voice.page.field.enableVoiceMode": "Увімкнути голосовий режим",
"settings.voice.page.field.provider": "Провайдер",
"settings.voice.page.provider.browser": "Браузер",
"settings.voice.page.provider.custom": "Власний",
"settings.voice.page.provider.say": "Say",
"settings.voice.page.provider.server": "Сервер",
"settings.voice.page.provider.local": "Локальний",
"settings.voice.page.tooltip.sttLocal": "Локальна розшифровка на сервері OpenChamber. Моделі завантажуються автоматично; ключ API не потрібен.",
"settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro, англійська). Модель завантажується автоматично; ключ API не потрібен.",
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (англійська)",
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 європейських мов)",
"settings.voice.page.stt.model.whisperBase": "Whisper base (мультимовна)",
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (мультимовна)",
"settings.voice.page.stt.badge.bestForEnglish": "Найкраща для англійської",
"settings.voice.page.stt.badge.bestForMultilingual": "Найкраща мультимовна",
"settings.voice.page.stt.meta.accuracy": "Точність",
"settings.voice.page.stt.meta.speed": "Швидкість",
"settings.voice.page.stt.modelInstalled": "Модель встановлено",
"settings.voice.page.stt.modelDownloading": "Завантаження моделі...",
"settings.voice.page.stt.modelDownloadingProgress": "Завантаження моделі... {percent}%",
"settings.voice.page.stt.modelNotInstalled": "Модель не завантажено",
"settings.voice.page.stt.modelDownload": "Завантажити",
"settings.voice.page.stt.modelDelete": "Видалити модель",
"settings.voice.page.stt.modelRetry": "Повторити",
"settings.voice.page.provider.wasm": "Локально",
"settings.voice.page.stt.wasmModel": "Модель Whisper",
"settings.voice.page.stt.wasmLoaded": "Модель завантажено",
+12
View File
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
'diffView.reviewDialog.actions.starting': 'Запуск...',
'diffView.reviewDialog.toast.noSessionDirectory': 'Директорія сесії недоступна',
'diffView.reviewDialog.toast.startFailed': 'Не вдалося запустити review flow',
'chat.dictation.start': 'Почати диктування',
'chat.dictation.overlayAria': 'Диктування',
'chat.dictation.downloadingModel': 'Завантаження мовної моделі...',
'chat.dictation.downloadingModelProgress': 'Завантаження мовної моделі... {percent}%',
'chat.dictation.listening': 'Слухаю...',
'chat.dictation.processing': 'Розшифровка...',
'chat.dictation.failed': 'Не вдалося розшифрувати запис',
'chat.dictation.cancel': 'Відхилити диктування',
'chat.dictation.insert': 'Вставити текст',
'chat.dictation.insertAndSend': 'Вставити й надіслати',
'chat.dictation.retry': 'Повторити розшифровку',
'chat.dictation.discard': 'Відхилити запис',
'chat.history.loadOlder': 'Завантажити ще',
'chat.autoReview.title': 'Цикл код-ревʼю триває',
'chat.autoReview.status.waitingForReviewer': 'Очікуємо ревʼювера',
@@ -1485,16 +1485,33 @@ export const settingsDict = {
'settings.notifications.page.toast.backgroundDisabled': '后台通知已关闭',
'settings.notifications.page.testNotification.title': '测试通知',
'settings.notifications.page.testNotification.body': '这是来自 OpenChamber 的测试通知。',
'settings.voice.page.section.voiceSetup': '语音设置',
'settings.voice.page.section.speechRecognition': '语音识别',
'settings.voice.page.field.enableVoiceInput': '启用语音输入',
'settings.voice.page.field.enableVoiceInputAria': '启用语音输入(听写)',
'settings.voice.page.section.playbackAndSummary': '播放',
'settings.voice.page.field.enableVoiceModeAria': '启用语音模式',
'settings.voice.page.field.enableVoiceMode': '启用语音模式',
'settings.voice.page.field.provider': '提供方',
'settings.voice.page.provider.browser': '浏览器',
'settings.voice.page.provider.custom': '自定义',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': '服务器',
'settings.voice.page.provider.local': '本地',
'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 服务器上本地转写。模型自动下载,无需 API 密钥。',
'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音(Kokoro,英语)。模型自动下载,无需 API 密钥。',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英语)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v325 种欧洲语言)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多语言)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多语言)',
'settings.voice.page.stt.badge.bestForEnglish': '英语最佳',
'settings.voice.page.stt.badge.bestForMultilingual': '多语言最佳',
'settings.voice.page.stt.meta.accuracy': '准确度',
'settings.voice.page.stt.meta.speed': '速度',
'settings.voice.page.stt.modelInstalled': '模型已安装',
'settings.voice.page.stt.modelDownloading': '正在下载模型...',
'settings.voice.page.stt.modelDownloadingProgress': '正在下载模型... {percent}%',
'settings.voice.page.stt.modelNotInstalled': '模型未下载',
'settings.voice.page.stt.modelDownload': '下载',
'settings.voice.page.stt.modelDelete': '删除模型',
'settings.voice.page.stt.modelRetry': '重试',
'settings.voice.page.provider.wasm': '本地',
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
'settings.voice.page.stt.wasmLoaded': '模型已加载',
@@ -1839,6 +1839,18 @@ export const dict: Record<I18nKey, string> = {
'chat.chatInput.placeholder.chat': '@ 用于文件/智能体;/ 用于命令和技能;! 用于 shell;# 用于代码片段',
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 辅助',
'chat.chatInput.placeholder.selectSession': '选择或创建会话以开始聊天',
'chat.dictation.start': '开始语音输入',
'chat.dictation.overlayAria': '语音输入',
'chat.dictation.downloadingModel': '正在下载语音模型...',
'chat.dictation.downloadingModelProgress': '正在下载语音模型... {percent}%',
'chat.dictation.listening': '正在聆听...',
'chat.dictation.processing': '正在转写...',
'chat.dictation.failed': '转写失败',
'chat.dictation.cancel': '放弃语音输入',
'chat.dictation.insert': '插入文本',
'chat.dictation.insertAndSend': '插入并发送',
'chat.dictation.retry': '重试转写',
'chat.dictation.discard': '放弃录音',
'chat.snippetAutocomplete.action.addNew': '+ 新建代码片段',
'chat.snippetAutocomplete.empty': '未找到代码片段',
'chat.snippetAutocomplete.footer': '↑↓ 导航 • Enter 选择 • Esc 关闭',
@@ -1401,16 +1401,33 @@
'settings.notifications.page.toast.backgroundDisabled': '背景通知已關閉',
'settings.notifications.page.testNotification.title': '測試通知',
'settings.notifications.page.testNotification.body': '這是來自 OpenChamber 的測試通知。',
'settings.voice.page.section.voiceSetup': '語音設定',
'settings.voice.page.section.speechRecognition': '語音辨識',
'settings.voice.page.field.enableVoiceInput': '啟用語音輸入',
'settings.voice.page.field.enableVoiceInputAria': '啟用語音輸入(聽寫)',
'settings.voice.page.section.playbackAndSummary': '播放',
'settings.voice.page.field.enableVoiceModeAria': '啟用語音模式',
'settings.voice.page.field.enableVoiceMode': '啟用語音模式',
'settings.voice.page.field.provider': '提供方',
'settings.voice.page.provider.browser': '瀏覽器',
'settings.voice.page.provider.custom': '自訂',
'settings.voice.page.provider.say': 'Say',
'settings.voice.page.provider.server': '伺服器',
'settings.voice.page.provider.local': '本機',
'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 伺服器上本機轉寫。模型會自動下載,無需 API 金鑰。',
'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音(Kokoro,英文)。模型會自動下載,無需 API 金鑰。',
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英文)',
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v325 種歐洲語言)',
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多語言)',
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多語言)',
'settings.voice.page.stt.badge.bestForEnglish': '英文最佳',
'settings.voice.page.stt.badge.bestForMultilingual': '多語言最佳',
'settings.voice.page.stt.meta.accuracy': '準確度',
'settings.voice.page.stt.meta.speed': '速度',
'settings.voice.page.stt.modelInstalled': '模型已安裝',
'settings.voice.page.stt.modelDownloading': '正在下載模型...',
'settings.voice.page.stt.modelDownloadingProgress': '正在下載模型... {percent}%',
'settings.voice.page.stt.modelNotInstalled': '模型未下載',
'settings.voice.page.stt.modelDownload': '下載',
'settings.voice.page.stt.modelDelete': '刪除模型',
'settings.voice.page.stt.modelRetry': '重試',
'settings.voice.page.provider.wasm': '本機',
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
'settings.voice.page.stt.wasmLoaded': '模型已載入',
@@ -1843,6 +1843,18 @@ export const dict: Record<I18nKey, string> = {
'chat.chatInput.placeholder.chat': '@ 用於檔案/Agent/ 用於命令;! 用於 shell;# 用於程式片段',
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 輔助',
'chat.chatInput.placeholder.selectSession': '選擇或建立會話以開始聊天',
'chat.dictation.start': '開始語音輸入',
'chat.dictation.overlayAria': '語音輸入',
'chat.dictation.downloadingModel': '正在下載語音模型...',
'chat.dictation.downloadingModelProgress': '正在下載語音模型... {percent}%',
'chat.dictation.listening': '正在聆聽...',
'chat.dictation.processing': '正在轉寫...',
'chat.dictation.failed': '轉寫失敗',
'chat.dictation.cancel': '放棄語音輸入',
'chat.dictation.insert': '插入文字',
'chat.dictation.insertAndSend': '插入並傳送',
'chat.dictation.retry': '重試轉寫',
'chat.dictation.discard': '放棄錄音',
'chat.snippetAutocomplete.action.addNew': '+ 新建程式片段',
'chat.snippetAutocomplete.empty': '未找到程式片段',
'chat.snippetAutocomplete.footer': '↑↓ 導航 • Enter 選擇 • Esc 關閉',
+26 -30
View File
@@ -113,7 +113,10 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
localStorage.removeItem('opencode-update-toast-dismissed-version');
}
}
if (settings.sttProvider === 'browser' || settings.sttProvider === 'server') {
if (typeof settings.dictationEnabled === 'boolean') {
localStorage.setItem('dictationEnabled', String(settings.dictationEnabled));
}
if (settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') {
localStorage.setItem('sttProvider', settings.sttProvider);
}
if (typeof settings.sttServerUrl === 'string') {
@@ -122,18 +125,12 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
if (typeof settings.sttModel === 'string') {
localStorage.setItem('sttModel', settings.sttModel);
}
if (typeof settings.sttLocalModel === 'string') {
localStorage.setItem('sttLocalModel', settings.sttLocalModel);
}
if (typeof settings.sttLanguage === 'string') {
localStorage.setItem('sttLanguage', settings.sttLanguage);
}
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb)) {
localStorage.setItem('sttSilenceThresholdDb', String(settings.sttSilenceThresholdDb));
}
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs)) {
localStorage.setItem('sttSilenceHoldMs', String(settings.sttSilenceHoldMs));
}
if (typeof settings.sttTranscribeOnStop === 'boolean') {
localStorage.setItem('sttTranscribeOnStop', String(settings.sttTranscribeOnStop));
}
};
const dispatchSettingsSynced = (settings: DesktopSettings): void => {
@@ -614,7 +611,10 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
}
if (configStoreApi && configStore) {
const nextConfigState: Partial<typeof configStore> = {};
if ((settings.sttProvider === 'browser' || settings.sttProvider === 'server') && settings.sttProvider !== configStore.sttProvider) {
if (typeof settings.dictationEnabled === 'boolean' && settings.dictationEnabled !== configStore.dictationEnabled) {
nextConfigState.dictationEnabled = settings.dictationEnabled;
}
if ((settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') && settings.sttProvider !== configStore.sttProvider) {
nextConfigState.sttProvider = settings.sttProvider;
}
if (typeof settings.sttServerUrl === 'string' && settings.sttServerUrl !== configStore.sttServerUrl) {
@@ -623,18 +623,12 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
if (typeof settings.sttModel === 'string' && settings.sttModel !== configStore.sttModel) {
nextConfigState.sttModel = settings.sttModel;
}
if (typeof settings.sttLocalModel === 'string' && settings.sttLocalModel !== configStore.sttLocalModel) {
nextConfigState.sttLocalModel = settings.sttLocalModel;
}
if (typeof settings.sttLanguage === 'string' && settings.sttLanguage !== configStore.sttLanguage) {
nextConfigState.sttLanguage = settings.sttLanguage;
}
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb) && settings.sttSilenceThresholdDb !== configStore.sttSilenceThresholdDb) {
nextConfigState.sttSilenceThresholdDb = settings.sttSilenceThresholdDb;
}
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs) && settings.sttSilenceHoldMs !== configStore.sttSilenceHoldMs) {
nextConfigState.sttSilenceHoldMs = settings.sttSilenceHoldMs;
}
if (typeof settings.sttTranscribeOnStop === 'boolean' && settings.sttTranscribeOnStop !== configStore.sttTranscribeOnStop) {
nextConfigState.sttTranscribeOnStop = settings.sttTranscribeOnStop;
}
if (Object.keys(nextConfigState).length > 0) {
configStoreApi.setState(nextConfigState);
}
@@ -1200,8 +1194,16 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
if (typeof candidate.responseStyleCustomInstructions === 'string') {
result.responseStyleCustomInstructions = candidate.responseStyleCustomInstructions;
}
if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'server') {
if (typeof candidate.dictationEnabled === 'boolean') {
result.dictationEnabled = candidate.dictationEnabled;
}
if (candidate.sttProvider === 'local' || candidate.sttProvider === 'openai-compatible') {
result.sttProvider = candidate.sttProvider;
} else if (candidate.sttProvider === 'server') {
// Legacy provider migration: 'server' was the OpenAI-compatible endpoint.
result.sttProvider = 'openai-compatible';
} else if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'wasm') {
result.sttProvider = 'local';
}
if (typeof candidate.sttServerUrl === 'string') {
result.sttServerUrl = candidate.sttServerUrl.trim();
@@ -1209,18 +1211,12 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
if (typeof candidate.sttModel === 'string') {
result.sttModel = candidate.sttModel.trim();
}
if (typeof candidate.sttLocalModel === 'string') {
result.sttLocalModel = candidate.sttLocalModel.trim();
}
if (typeof candidate.sttLanguage === 'string') {
result.sttLanguage = candidate.sttLanguage.trim();
}
if (typeof candidate.sttSilenceThresholdDb === 'number' && Number.isFinite(candidate.sttSilenceThresholdDb)) {
result.sttSilenceThresholdDb = candidate.sttSilenceThresholdDb;
}
if (typeof candidate.sttSilenceHoldMs === 'number' && Number.isFinite(candidate.sttSilenceHoldMs)) {
result.sttSilenceHoldMs = candidate.sttSilenceHoldMs;
}
if (typeof candidate.sttTranscribeOnStop === 'boolean') {
result.sttTranscribeOnStop = candidate.sttTranscribeOnStop;
}
return result;
};
+4 -10
View File
@@ -678,22 +678,16 @@ const SETTINGS_SEARCH_ITEMS: readonly SettingsSearchItem[] = [
keywords: ['keyboard', 'hotkeys', 'bindings'],
},
{
id: 'voice.voice-setup',
id: 'voice.playback',
page: 'voice',
titleKey: 'settings.voice.page.section.voiceSetup',
keywords: ['tts', 'voice mode', 'provider', 'speech rate', 'speech pitch', 'speech volume', 'language'],
titleKey: 'settings.voice.page.section.playbackAndSummary',
keywords: ['tts', 'read aloud', 'voice', 'provider', 'speech rate', 'speech pitch', 'speech volume', 'tts input mode', 'markdown'],
},
{
id: 'voice.speech-recognition',
page: 'voice',
titleKey: 'settings.voice.page.section.speechRecognition',
keywords: ['stt', 'transcribe', 'whisper', 'microphone', 'silence threshold'],
},
{
id: 'voice.playback',
page: 'voice',
titleKey: 'settings.voice.page.section.playbackAndSummary',
keywords: ['read aloud', 'tts input mode', 'summary', 'markdown'],
keywords: ['stt', 'dictation', 'voice input', 'transcribe', 'whisper', 'parakeet', 'microphone'],
},
{
id: 'tunnel.provider',
+7
View File
@@ -327,6 +327,13 @@ const SHORTCUT_ACTIONS: ReadonlyArray<ShortcutAction> = [
description: 'Toggle focus mode for the chat input',
customizable: true,
},
{
id: 'toggle_dictation',
defaultCombo: 'mod+alt+v',
label: 'Voice input',
description: 'Start dictation; press again to confirm and insert the transcript',
customizable: true,
},
{
id: 'abort_run',
defaultCombo: 'escape',
@@ -1,397 +0,0 @@
/**
* Audio Stream Service
*
* Captures microphone audio using MediaRecorder, detects utterance boundaries
* via an AnalyserNode-based silence detector (VAD), then POSTs each utterance
* as a raw audio blob to the OpenChamber server's /api/stt/transcribe endpoint.
*
* Mimics the BrowserVoiceService.startListening interface so useBrowserVoice
* can swap providers without changing its internal logic.
*
* @example
* ```typescript
* audioStreamService.configure({ baseURL: 'http://localhost:8001/v1', model: 'whisper-1' });
* audioStreamService.startListening('en', (text, isFinal) => {
* if (isFinal) console.log('transcript:', text);
* });
* audioStreamService.stopListening();
* ```
*/
import { runtimeFetch } from '@/lib/runtime-fetch';
type SpeechResultCallback = (text: string, isFinal: boolean) => void;
type ErrorCallback = (error: string) => void;
interface AudioStreamConfig {
/** Base URL of the OpenAI-compatible STT server (e.g. http://localhost:8001/v1) */
baseURL: string;
/** Whisper-compatible model name */
model: string;
/** Optional BCP-47 language hint (e.g. 'en'). Empty string = auto-detect. */
language?: string;
/**
* Silence threshold in dB below which audio is considered silence.
* Lower (more negative) = only very quiet audio counts as silence.
* Default: -45
*/
silenceThresholdDb?: number;
/**
* How long continuous silence must last (ms) before the utterance is finalised.
* Default: 1500
*/
silenceHoldMs?: number;
/** Optional API key for the STT server. */
apiKey?: string;
}
// How often (ms) the VAD samples the analyser
const VAD_POLL_MS = 80;
// Minimum audio duration (ms) to bother uploading (avoids blank clips)
const MIN_UTTERANCE_MS = 300;
class AudioStreamService {
private stream: MediaStream | null = null;
private mediaRecorder: MediaRecorder | null = null;
private audioContext: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private vadTimer: ReturnType<typeof setInterval> | null = null;
private chunks: Blob[] = [];
private recordingStartMs = 0;
private isActive = false;
private isSpeaking = false;
private silenceSince: number | null = null;
private onResult: SpeechResultCallback | null = null;
private onError: ErrorCallback | null = null;
private finishResolver: (() => void) | null = null;
private lang = 'en';
// Configurable parameters
private cfg: Required<AudioStreamConfig> = {
baseURL: '',
model: 'deepdml/faster-whisper-large-v3-turbo-ct2',
language: '',
silenceThresholdDb: -45,
silenceHoldMs: 1500,
apiKey: '',
};
/** Update service configuration. Can be called before or after startListening. */
configure(config: AudioStreamConfig): void {
this.cfg = {
silenceThresholdDb: -45,
silenceHoldMs: 1500,
language: '',
apiKey: '',
...config,
};
this.cfg.apiKey = config.apiKey ?? '';
}
/** Whether the browser supports the required APIs. */
isSupported(): boolean {
return (
typeof window !== 'undefined' &&
typeof navigator !== 'undefined' &&
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
typeof window.MediaRecorder !== 'undefined' &&
typeof window.AudioContext !== 'undefined'
);
}
/**
* Start listening. Requests microphone access if not already held.
* Calls onResult(text, true) for each completed utterance.
*/
async startListening(
lang: string,
onResult: SpeechResultCallback,
onError?: ErrorCallback
): Promise<void> {
if (this.isActive) {
this.stopListening();
}
this.lang = lang;
this.onResult = onResult;
this.onError = onError ?? null;
this.isActive = true;
try {
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
} catch (err) {
this.isActive = false;
const msg = err instanceof Error ? err.message : 'Microphone access denied';
onError?.(msg);
return;
}
this._setupAudioContext();
this._startRecorder();
this._startVAD();
}
/** Stop listening and clean up all resources. */
stopListening(): void {
this._stopVAD();
this._stopRecorder();
this._cleanupAfterStop(true);
}
async finishListening(): Promise<void> {
if (!this.isActive) return;
this._stopVAD();
this.isSpeaking = false;
this.silenceSince = null;
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
this._cleanupAfterStop(true);
return;
}
await new Promise<void>((resolve) => {
this.finishResolver = resolve;
this._finaliseUtterance(false);
});
this._cleanupAfterStop(true);
}
/** Whether currently listening. */
getIsListening(): boolean {
return this.isActive;
}
// ── Private helpers ──────────────────────────────────────────────────────
private _setupAudioContext(): void {
if (!this.stream) return;
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
this.audioContext = new AudioContextClass();
const source = this.audioContext.createMediaStreamSource(this.stream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 512;
source.connect(this.analyser);
}
private _teardownAudioContext(): void {
try {
this.audioContext?.close();
} catch {
// ignore
}
this.audioContext = null;
this.analyser = null;
}
private _startRecorder(): void {
if (!this.stream) return;
const mimeType = this._pickMimeType();
const options: MediaRecorderOptions = {};
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
options.mimeType = mimeType;
}
this.mediaRecorder = new MediaRecorder(this.stream, options);
this.chunks = [];
this.recordingStartMs = Date.now();
this.mediaRecorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) {
this.chunks.push(e.data);
}
};
this.mediaRecorder.onstop = () => {
const blobs = this.chunks.splice(0);
const durationMs = Date.now() - this.recordingStartMs;
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
this.finishResolver?.();
this.finishResolver = null;
return;
}
const mType = blobs[0].type || mimeType || 'audio/webm';
const blob = new Blob(blobs, { type: mType });
void this._upload(blob, mType).finally(() => {
this.finishResolver?.();
this.finishResolver = null;
});
};
// Collect data every 250 ms so we don't lose the tail on stop()
this.mediaRecorder.start(250);
}
private _stopRecorder(): void {
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
try {
this.mediaRecorder.stop();
} catch {
this.finishResolver?.();
this.finishResolver = null;
// ignore
}
}
this.mediaRecorder = null;
}
private _releaseStream(): void {
if (this.stream) {
this.stream.getTracks().forEach((t) => t.stop());
this.stream = null;
}
}
private _startVAD(): void {
this._stopVAD();
this.silenceSince = null;
this.isSpeaking = false;
this.vadTimer = setInterval(() => {
if (!this.isActive || !this.analyser) return;
const db = this._getRmsDb();
const isSilent = db < this.cfg.silenceThresholdDb;
if (!isSilent) {
// Audio detected
this.silenceSince = null;
if (!this.isSpeaking) {
this.isSpeaking = true;
// Restart recorder to capture from the start of speech
if (this.mediaRecorder?.state === 'recording') {
this.recordingStartMs = Date.now();
}
}
} else {
// Silence detected
if (this.isSpeaking) {
if (this.silenceSince === null) {
this.silenceSince = Date.now();
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
// End of utterance — stop recorder (triggers onstop → upload)
this.isSpeaking = false;
this.silenceSince = null;
this._finaliseUtterance(true);
}
}
}
}, VAD_POLL_MS);
}
private _stopVAD(): void {
if (this.vadTimer !== null) {
clearInterval(this.vadTimer);
this.vadTimer = null;
}
}
private _cleanupAfterStop(clearChunks: boolean): void {
const pendingResolver = this.finishResolver;
this.isActive = false;
this.finishResolver = null;
this.mediaRecorder = null;
this._teardownAudioContext();
this._releaseStream();
if (clearChunks) {
this.chunks = [];
}
this.isSpeaking = false;
this.silenceSince = null;
this.onResult = null;
this.onError = null;
pendingResolver?.();
}
/** Stop the current recorder to flush the utterance, optionally restarting for the next one. */
private _finaliseUtterance(restart: boolean): void {
if (!this.isActive) return;
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
this.mediaRecorder.stop();
}
if (!restart) return;
// Restart recorder for the next utterance after a short delay
// (MediaRecorder.onstop fires asynchronously; we wait for it to complete)
setTimeout(() => {
if (this.isActive && this.stream) {
this._startRecorder();
}
}, 100);
}
/** Compute RMS of current analyser frame in dBFS. */
private _getRmsDb(): number {
if (!this.analyser) return -Infinity;
const buf = new Float32Array(this.analyser.fftSize);
this.analyser.getFloatTimeDomainData(buf);
let sumSq = 0;
for (const s of buf) sumSq += s * s;
const rms = Math.sqrt(sumSq / buf.length);
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
}
/** POST utterance blob to server, call onResult with transcript. */
private async _upload(blob: Blob, mimeType: string): Promise<void> {
if (!this.onResult) return;
try {
const headers: Record<string, string> = {
'Content-Type': mimeType,
'X-Base-URL': this.cfg.baseURL,
'X-Model': this.cfg.model,
};
if (this.cfg.apiKey) {
headers['Authorization'] = `Bearer ${this.cfg.apiKey}`;
}
if (this.cfg.language) {
headers['X-Language'] = this.cfg.language;
} else if (this.lang && this.lang !== 'auto') {
// Use BCP-47 base language code (e.g. 'en' from 'en-US')
const baseLang = this.lang.split('-')[0];
headers['X-Language'] = baseLang;
}
const response = await runtimeFetch('/api/stt/transcribe', {
method: 'POST',
headers,
body: blob,
});
if (!response.ok) {
const errData = await response.json().catch(() => ({ error: 'Unknown error' }));
throw new Error(errData.error ?? `HTTP ${response.status}`);
}
const data = await response.json();
const transcript: string = (data.transcript ?? '').trim();
if (transcript) {
this.onResult(transcript, true);
}
} catch (err) {
if (!this.isActive) return; // Stopped — ignore
const msg = err instanceof Error ? err.message : 'Transcription upload failed';
console.error('[AudioStreamService] Upload error:', msg);
this.onError?.(msg);
}
}
/** Pick the best supported MIME type for MediaRecorder. */
private _pickMimeType(): string {
const candidates = [
'audio/webm;codecs=opus',
'audio/webm',
'audio/ogg;codecs=opus',
'audio/ogg',
'audio/mp4',
];
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
}
return '';
}
}
export const audioStreamService = new AudioStreamService();
@@ -1,131 +0,0 @@
/**
* Context formatters for voice-native output
* Formats session events (messages, permissions, ready events) into natural language
* for the ElevenLabs voice agent to speak aloud.
*
* @example
* ```typescript
* import { formatMessage, formatPermissionRequest } from '@/lib/voice';
*
* const voiceText = formatMessage({ role: 'assistant', content: 'Hello!' });
* // Returns: "Claude Code: Hello!"
* ```
*/
import { VOICE_CONFIG } from "./voiceConfig";
/** Message type for voice formatting */
export interface VoiceMessage {
role: string;
content: string;
}
/**
* Format a single message for voice output
* - Assistant messages: Code blocks replaced with "[code block]", prefixed with "Claude Code: "
* - User messages: Prefixed with "User: "
* - Other roles: Returns null (not spoken)
*
* @param message - The message to format
* @returns Formatted text for voice, or null if should not be spoken
*/
function formatMessage(message: VoiceMessage): string | null {
// Handle edge cases
if (!message || typeof message.content !== "string") {
return null;
}
const content = message.content.trim();
if (!content) {
return null;
}
if (message.role === "assistant") {
// Replace code blocks with description (don't read code aloud)
const textOnly = content.replace(/```[\s\S]*?```/g, "[code block]");
return `Claude Code: ${textOnly}`;
}
if (message.role === "user") {
return `User: ${content}`;
}
// Skip system, tool, and other roles for voice
return null;
}
/**
* Format multiple new messages for voice output
* - Maps messages through formatMessage
* - Filters out nulls (unspoken roles)
* - Joins with newlines
*
* @param sessionId - The session ID (for future use/debugging)
* @param messages - Array of messages to format
* @returns Formatted text for voice, or null if no speakable messages
*/
export function formatNewMessages(
sessionId: string,
messages: VoiceMessage[]
): string | null {
// Handle edge cases
if (!Array.isArray(messages) || messages.length === 0) {
return null;
}
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log(`[Voice] Formatting ${messages.length} messages for session ${sessionId}`);
}
// Format each message and filter out nulls
const formattedMessages = messages
.map(formatMessage)
.filter((msg): msg is string => msg !== null);
if (formattedMessages.length === 0) {
return null;
}
return formattedMessages.join("\n");
}
/**
* Format a permission request for voice announcement
* - Per CONTEXT.md: Only tool name, not arguments (LIMITED_TOOL_CALLS)
* - Prompts user to say "allow" or "deny"
*
* @param sessionId - The session ID
* @param requestId - The permission request ID
* @param toolName - Name of the tool requesting permission
* @param toolArgs - Tool arguments (not included in voice output per config)
* @returns Formatted permission request for voice
*/
export function formatPermissionRequest(
sessionId: string,
requestId: string,
toolName: string,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
toolArgs: unknown
): string {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log(`[Voice] Formatting permission request ${requestId} for session ${sessionId}`);
}
// Per VOICE_CONFIG.LIMITED_TOOL_CALLS, we don't include toolArgs in voice output
return `Claude Code is requesting permission to use ${toolName}. Say "allow" or "deny".`;
}
/**
* Format a ready event for voice announcement
* - Indicates the AI has finished working and is ready for next instruction
*
* @param sessionId - The session ID
* @returns Formatted ready event for voice
*/
export function formatReadyEvent(sessionId: string): string {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log(`[Voice] Formatting ready event for session ${sessionId}`);
}
return "Claude Code finished working. Ready for next instruction.";
}
-17
View File
@@ -1,17 +0,0 @@
/**
* Voice module barrel export
* Provides a clean import path for voice session hooks.
*
* @example
* ```typescript
* import { voiceHooks } from '@/lib/voice';
* ```
*/
// Voice session registry (from voiceSession.ts)
export {
isVoiceSessionStarted,
} from "./voiceSession";
// Voice hooks for session-to-voice event routing (from voiceHooks.ts)
export { voiceHooks } from "./voiceHooks";
+10 -4
View File
@@ -6,15 +6,21 @@
export function sanitizeForTTS(text: string): string {
if (!text) return '';
return text
// Remove code blocks
// Remove fenced code blocks entirely (multi-line code is unreadable),
// but keep inline-code CONTENT and only strip the backticks: agents
// routinely inline meaningful words ("You are on `main`").
.replace(/```[\s\S]*?```/g, '')
.replace(/`[^`]*`/g, '')
.replace(/`([^`\n]*)`/g, '$1')
// Remove markdown formatting
.replace(/[*_~#]/g, '')
// Remove URLs
.replace(/https?:\/\/[^\s]+/g, '')
// Remove file paths
.replace(/\/[\w\-./]+/g, '')
// Remove absolute file paths (leading slash, one or more segments).
// Deliberately NOT matching interword slashes: "iOS/Android" and
// "origin/main" are speech, not paths.
.replace(/(^|\s)\/(?:[\w.-]+\/)*[\w.-]+/g, '$1')
// Read remaining interword slashes out loud ("iOS slash Android").
.replace(/([\w.])\/([\w.])/g, '$1 slash $2')
// Remove shell-like patterns
.replace(/^\s*[$#>]\s*/gm, '')
// Remove brackets and special chars
-32
View File
@@ -1,32 +0,0 @@
/**
* Static voice context configuration
* Controls voice behavior and feature flags for the ElevenLabs voice agent
*/
export const VOICE_CONFIG = {
/** Disable all tool call information from being sent to voice context */
DISABLE_TOOL_CALLS: false,
/** Send only tool names and descriptions, exclude arguments */
LIMITED_TOOL_CALLS: true,
/** Disable permission request forwarding */
DISABLE_PERMISSION_REQUESTS: false,
/** Disable session online/offline notifications */
DISABLE_SESSION_STATUS: true,
/** Disable message forwarding */
DISABLE_MESSAGES: false,
/** Disable session focus notifications */
DISABLE_SESSION_FOCUS: false,
/** Disable ready event notifications */
DISABLE_READY_EVENTS: false,
/** Maximum number of messages to include in session history */
MAX_HISTORY_MESSAGES: 50,
/** Enable debug logging for voice context updates */
ENABLE_DEBUG_LOGGING: true,
} as const;
-125
View File
@@ -1,125 +0,0 @@
/**
* Voice hooks for session-to-voice event routing
* Routes session events (messages, permissions, ready events) to the ElevenLabs
* voice agent via contextual updates.
*
* This module provides hooks that can be called when session events occur,
* using the voice session registry from voiceSession.ts.
*
* @example
* ```typescript
* import { voiceHooks } from '@/lib/voice';
*
* // Route session messages to voice
* voiceHooks.onMessages(sessionId, messages);
* ```
*/
import { VOICE_CONFIG } from "./voiceConfig";
import {
formatNewMessages,
formatPermissionRequest,
formatReadyEvent,
type VoiceMessage,
} from "./contextFormatters";
import { getVoiceSession, isVoiceSessionStarted } from "./voiceSession";
/**
* Report a contextual update to the voice session
* Internal helper that checks preconditions and handles errors
*
* @param update - The text update to send, or null/undefined to skip
*/
function reportContextualUpdate(update: string | null | undefined): void {
// Skip empty/null/undefined updates
if (!update || update.trim().length === 0) {
return;
}
// Skip if no voice session or not started
const voiceSession = getVoiceSession();
if (!voiceSession || !isVoiceSessionStarted()) {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log("[Voice] Skipping contextual update - no active session");
}
return;
}
try {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log("[Voice] Sending contextual update:", update.substring(0, 100));
}
voiceSession.sendContextualUpdate(update);
} catch (error) {
// Log error but don't throw - voice updates shouldn't break the app
console.error("[Voice] Failed to send contextual update:", error);
}
}
/**
* Voice hooks - exported functions to route session events to voice
*
* These hooks should be called when corresponding session events occur.
* They respect VOICE_CONFIG feature flags to enable/disable specific
* event types.
*/
export const voiceHooks = {
/**
* Called when new messages arrive in the session
* Formats and sends messages to voice agent (if not disabled)
*
* @param sessionId - The session ID
* @param messages - Array of messages to format and send
*/
onMessages(sessionId: string, messages: VoiceMessage[]): void {
if (VOICE_CONFIG.DISABLE_MESSAGES) {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log("[Voice] Message forwarding disabled");
}
return;
}
reportContextualUpdate(formatNewMessages(sessionId, messages));
},
/**
* Called when a permission request is made
* Announces the permission request to the voice agent (if not disabled)
*
* @param sessionId - The session ID
* @param requestId - The permission request ID
* @param toolName - Name of the tool requesting permission
* @param toolArgs - Arguments for the tool (not sent to voice per LIMITED_TOOL_CALLS)
*/
onPermissionRequested(
sessionId: string,
requestId: string,
toolName: string,
toolArgs: unknown
): void {
if (VOICE_CONFIG.DISABLE_PERMISSION_REQUESTS) {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log("[Voice] Permission request forwarding disabled");
}
return;
}
reportContextualUpdate(
formatPermissionRequest(sessionId, requestId, toolName, toolArgs)
);
},
/**
* Called when the AI is ready for the next instruction
* Announces ready state to the voice agent (if not disabled)
*
* @param sessionId - The session ID
*/
onReady(sessionId: string): void {
if (VOICE_CONFIG.DISABLE_READY_EVENTS) {
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
console.log("[Voice] Ready event forwarding disabled");
}
return;
}
reportContextualUpdate(formatReadyEvent(sessionId));
},
};
-28
View File
@@ -1,28 +0,0 @@
/**
* Voice session interface
* Used for type safety without importing ReturnType from SDK
*/
interface VoiceSession {
sendContextualUpdate: (text: string) => void;
}
/**
* Global storage for the active voice session.
* Used by voiceHooks to send contextual updates to the voice agent.
*/
const activeVoiceSession: VoiceSession | null = null;
/**
* Get the currently registered voice session.
* Used by voiceHooks to send contextual updates.
*/
export function getVoiceSession(): VoiceSession | null {
return activeVoiceSession;
}
/**
* Check if a voice session is currently active.
*/
export function isVoiceSessionStarted(): boolean {
return activeVoiceSession !== null;
}
-555
View File
@@ -1,555 +0,0 @@
/**
* WASM Speech-to-Text Service
*
* Local Whisper transcription via Transformers.js (ONNX Runtime Web).
* Captures microphone audio, detects utterance boundaries via silence-based
* VAD, then transcribes each utterance locally no cloud API required.
*
* Works in Electron and all modern browsers that support Web Audio API.
* First use downloads a Whisper model (~40166 MB, cached).
*/
export type WasmModelStatus =
| { state: 'unloaded' }
| { state: 'downloading'; progress: number }
| { state: 'loading' }
| { state: 'ready' }
| { state: 'error'; error: string };
export interface WasmModelInfo {
id: string;
name: string;
size: string;
languages: string;
description: string;
}
export const WASM_MODELS: WasmModelInfo[] = [
{
id: 'Xenova/whisper-tiny.en',
name: 'Whisper Tiny (EN)',
size: '~39 MB',
languages: 'English',
description: 'Fastest, lowest accuracy. Good for quick dictation.',
},
{
id: 'Xenova/whisper-base.en',
name: 'Whisper Base (EN)',
size: '~73 MB',
languages: 'English',
description: 'Balanced speed and accuracy. Default for English.',
},
{
id: 'Xenova/whisper-small.en',
name: 'Whisper Small (EN)',
size: '~166 MB',
languages: 'English',
description: 'Higher accuracy, slower. Best for noisy environments.',
},
];
type SpeechResultCallback = (text: string, isFinal: boolean) => void;
type ErrorCallback = (error: string) => void;
const VAD_POLL_MS = 80;
const MIN_UTTERANCE_MS = 300;
const WHISPER_SAMPLE_RATE = 16000;
interface WasmSttConfig {
silenceThresholdDb?: number;
silenceHoldMs?: number;
}
class WasmSttService {
private transcriber: unknown = null;
private worker: Worker | null = null;
private modelStatus: WasmModelStatus = { state: 'unloaded' };
private currentModelId: string | null = null;
private stream: MediaStream | null = null;
private mediaRecorder: MediaRecorder | null = null;
private audioContext: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private vadTimer: ReturnType<typeof setInterval> | null = null;
private chunks: Blob[] = [];
private recordingStartMs = 0;
private isActive = false;
private isSpeaking = false;
private silenceSince: number | null = null;
private onResult: SpeechResultCallback | null = null;
private onError: ErrorCallback | null = null;
private finishResolver: (() => void) | null = null;
private lang = 'en';
private cfg: Required<WasmSttConfig> = {
silenceThresholdDb: -45,
silenceHoldMs: 1500,
};
public onModelStatusChange: ((status: WasmModelStatus) => void) | null = null;
configure(config: WasmSttConfig): void {
this.cfg = { ...this.cfg, ...config };
}
isSupported(): boolean {
return (
typeof window !== 'undefined' &&
typeof navigator !== 'undefined' &&
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
typeof window.MediaRecorder !== 'undefined' &&
typeof window.AudioContext !== 'undefined'
);
}
getModelStatus(): WasmModelStatus {
return this.modelStatus;
}
getCurrentModelId(): string | null {
return this.currentModelId;
}
private setModelStatus(status: WasmModelStatus): void {
this.modelStatus = status;
this.onModelStatusChange?.(status);
}
async loadModel(modelId: string): Promise<void> {
if (this.currentModelId === modelId && this.modelStatus.state === 'ready') {
return;
}
if (this.modelStatus.state === 'downloading' || this.modelStatus.state === 'loading') {
return;
}
this._terminateWorker();
this.transcriber = null;
this.setModelStatus({ state: 'downloading', progress: 0 });
this.currentModelId = modelId;
// Try Web Worker first — inference off main thread = no UI freeze.
try {
const WasmWorkerMod = await import('./wasmSttWorker?worker');
const WasmWorker = WasmWorkerMod.default as new () => Worker;
this.worker = new WasmWorker();
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('Worker init timed out')), 10000);
this.worker!.onmessage = (e: MessageEvent) => {
const data = e.data as { type: string; progress?: number; error?: string; text?: string };
if (data.type === 'progress') {
this.setModelStatus({ state: 'downloading', progress: data.progress ?? 0 });
} else if (data.type === 'loaded') {
clearTimeout(timer);
resolve();
} else if (data.type === 'error') {
clearTimeout(timer);
reject(new Error(data.error ?? 'Worker load failed'));
}
};
this.worker!.onerror = (err) => {
clearTimeout(timer);
reject(new Error(err.message || 'Worker error'));
};
this.worker!.postMessage({ type: 'load', modelId });
});
this.setModelStatus({ state: 'ready' });
return;
} catch (err) {
console.warn('[WasmStt] Worker failed, using main-thread:', err instanceof Error ? err.message : err);
this._terminateWorker();
}
// Fallback: main-thread pipeline (causes brief UI freeze during inference).
try {
const { pipeline, env } = await import('@xenova/transformers');
env.backends.onnx.wasm.numThreads = 1;
env.allowLocalModels = false;
const fileDoneBytes = new Map<string, number>();
let totalDone = 0;
let totalEstimate = 0;
this.transcriber = await pipeline('automatic-speech-recognition', modelId, {
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
if (info.status === 'progress' && info.file) {
const prevDone = fileDoneBytes.get(info.file) ?? 0;
const currentDone = info.loaded ?? 0;
const delta = Math.max(0, currentDone - prevDone);
fileDoneBytes.set(info.file, currentDone);
totalDone += delta;
if (info.total && info.total > totalEstimate) totalEstimate = info.total;
const effectiveTotal = Math.max(totalEstimate, totalDone);
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
this.setModelStatus({ state: 'downloading', progress: pct });
}
},
});
this.setModelStatus({ state: 'ready' });
} catch (err) {
const msg = err instanceof Error ? err.message : 'Unknown error loading model';
this.setModelStatus({ state: 'error', error: msg });
this.transcriber = null;
this.currentModelId = null;
throw err;
}
}
private _terminateWorker(): void {
if (this.worker) {
this.worker.terminate();
this.worker = null;
}
}
async unloadModel(): Promise<void> {
this._terminateWorker();
this.transcriber = null;
this.currentModelId = null;
this.setModelStatus({ state: 'unloaded' });
}
async startListening(
lang: string,
onResult: SpeechResultCallback,
onError?: ErrorCallback,
): Promise<void> {
if (this.isActive) {
this.stopListening();
}
if (!this.transcriber && !this.worker) {
onError?.('Whisper model not loaded. Select a model in Voice Settings first.');
return;
}
this.lang = lang;
this.onResult = onResult;
this.onError = onError ?? null;
this.isActive = true;
try {
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
} catch (err) {
this.isActive = false;
const msg = err instanceof Error ? err.message : 'Microphone access denied';
onError?.(msg);
return;
}
this._setupAudioContext();
this._startRecorder();
this._startVAD();
}
stopListening(): void {
this._stopVAD();
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
try { this.mediaRecorder.stop(); } catch { /* ignore */ }
}
this._cleanupAfterStop(true);
}
async finishListening(): Promise<void> {
if (!this.isActive) return;
this._stopVAD();
this.isSpeaking = false;
this.silenceSince = null;
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
this._cleanupAfterStop(true);
return;
}
await new Promise<void>((resolve) => {
this.finishResolver = resolve;
this._finaliseUtterance(false);
});
this._cleanupAfterStop(true);
}
getIsListening(): boolean {
return this.isActive;
}
// ── Audio capture ────────────────────────────────────────────────────
private _setupAudioContext(): void {
if (!this.stream) return;
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
this.audioContext = new AudioContextClass();
const source = this.audioContext.createMediaStreamSource(this.stream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 512;
source.connect(this.analyser);
}
private _teardownAudioContext(): void {
try { this.audioContext?.close(); } catch { /* ignore */ }
this.audioContext = null;
this.analyser = null;
}
private _startRecorder(): void {
if (!this.stream) return;
const mimeType = this._pickMimeType();
const options: MediaRecorderOptions = {};
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
options.mimeType = mimeType;
}
this.mediaRecorder = new MediaRecorder(this.stream, options);
this.chunks = [];
this.recordingStartMs = Date.now();
this.mediaRecorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) {
this.chunks.push(e.data);
}
};
this.mediaRecorder.onstop = () => {
const blobs = this.chunks.splice(0);
const durationMs = Date.now() - this.recordingStartMs;
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
this.finishResolver?.();
this.finishResolver = null;
return;
}
const mType = blobs[0].type || mimeType || 'audio/webm';
const blob = new Blob(blobs, { type: mType });
void this._transcribe(blob).finally(() => {
this.finishResolver?.();
this.finishResolver = null;
});
};
this.mediaRecorder.start(250);
}
private _releaseStream(): void {
if (this.stream) {
this.stream.getTracks().forEach((t) => t.stop());
this.stream = null;
}
}
// ── VAD ──────────────────────────────────────────────────────────────
private _startVAD(): void {
this._stopVAD();
this.silenceSince = null;
this.isSpeaking = false;
this.vadTimer = setInterval(() => {
if (!this.isActive || !this.analyser) return;
const db = this._getRmsDb();
const isSilent = db < this.cfg.silenceThresholdDb;
if (!isSilent) {
this.silenceSince = null;
if (!this.isSpeaking) {
this.isSpeaking = true;
if (this.mediaRecorder?.state === 'recording') {
this.recordingStartMs = Date.now();
}
}
} else {
if (this.isSpeaking) {
if (this.silenceSince === null) {
this.silenceSince = Date.now();
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
this.isSpeaking = false;
this.silenceSince = null;
this._finaliseUtterance(true);
}
}
}
}, VAD_POLL_MS);
}
private _stopVAD(): void {
if (this.vadTimer !== null) {
clearInterval(this.vadTimer);
this.vadTimer = null;
}
}
private _cleanupAfterStop(clearChunks: boolean): void {
const pendingResolver = this.finishResolver;
this.isActive = false;
this.finishResolver = null;
this.mediaRecorder = null;
this._teardownAudioContext();
this._releaseStream();
if (clearChunks) this.chunks = [];
this.isSpeaking = false;
this.silenceSince = null;
this.onResult = null;
this.onError = null;
pendingResolver?.();
}
private _finaliseUtterance(restart: boolean): void {
if (!this.isActive) return;
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
this.mediaRecorder.stop();
}
if (!restart) return;
setTimeout(() => {
if (this.isActive && this.stream) {
this._startRecorder();
}
}, 100);
}
private _getRmsDb(): number {
if (!this.analyser) return -Infinity;
const buf = new Float32Array(this.analyser.fftSize);
this.analyser.getFloatTimeDomainData(buf);
let sumSq = 0;
for (const s of buf) sumSq += s * s;
const rms = Math.sqrt(sumSq / buf.length);
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
}
// ── Transcription ────────────────────────────────────────────────────
private async _transcribe(blob: Blob): Promise<void> {
if (!this.onResult) return;
if (!this.transcriber && !this.worker) {
this.onError?.('Model not loaded');
return;
}
try {
const audioData = await this._decodeToFloat32(blob);
if (!audioData || audioData.length === 0) {
this.onError?.(`Failed to decode audio (${blob.size} bytes)`);
return;
}
const langHint = this._resolveLanguageHint();
// Prefer worker (non-blocking); fall back to main-thread pipeline.
const transcript = this.worker
? await this._transcribeViaWorker(audioData, langHint)
: await this._transcribeMainThread(audioData, langHint);
if (transcript) {
this.onResult(transcript, true);
}
} catch (err) {
if (!this.isActive) return;
const msg = err instanceof Error ? err.message : 'Local transcription failed';
this.onError?.(msg);
}
}
private _transcribeViaWorker(audioData: Float32Array, langHint: string | undefined): Promise<string> {
return new Promise((resolve, reject) => {
if (!this.worker) return reject(new Error('Worker gone'));
const onMessage = (e: MessageEvent) => {
const data = e.data as { type: string; error?: string; transcript?: string; text?: string };
if (data.type === 'result') {
this.worker!.removeEventListener('message', onMessage);
resolve(data.transcript ?? '');
} else if (data.type === 'log') {
console.log('[WasmStt Worker]', data.text);
} else if (data.type === 'error') {
this.worker!.removeEventListener('message', onMessage);
reject(new Error(data.error ?? 'Transcription failed'));
}
};
this.worker.addEventListener('message', onMessage);
this.worker.postMessage(
{ type: 'transcribe', audio: audioData.buffer, language: langHint },
[audioData.buffer],
);
setTimeout(() => {
this.worker?.removeEventListener('message', onMessage);
reject(new Error('Transcription timed out'));
}, 30000);
});
}
private async _transcribeMainThread(audioData: Float32Array, langHint: string | undefined): Promise<string> {
const pipelineFn = this.transcriber as (
input: Float32Array,
options?: Record<string, unknown>,
) => Promise<{ text: string }>;
const result = await pipelineFn(audioData, {
task: 'transcribe',
...(langHint ? { language: langHint } : {}),
});
return (result?.text ?? '').trim();
}
private async _decodeToFloat32(blob: Blob): Promise<Float32Array | null> {
if (!this.audioContext) return null;
const arrayBuffer = await blob.arrayBuffer();
let audioBuffer: AudioBuffer;
try {
audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
} catch {
return null;
}
const origRate = audioBuffer.sampleRate;
const origData = audioBuffer.getChannelData(0);
const targetRate = WHISPER_SAMPLE_RATE;
if (origRate === targetRate) {
return new Float32Array(origData);
}
const ratio = origRate / targetRate;
const newLength = Math.ceil(origData.length / ratio);
const result = new Float32Array(newLength);
for (let i = 0; i < newLength; i++) {
const origIdx = i * ratio;
const idx0 = Math.floor(origIdx);
const idx1 = Math.min(idx0 + 1, origData.length - 1);
const frac = origIdx - idx0;
result[i] = origData[idx0] * (1 - frac) + origData[idx1] * frac;
}
return result;
}
private _resolveLanguageHint(): string | undefined {
if (this.lang && this.lang !== 'auto') {
return this.lang.split('-')[0];
}
return undefined;
}
private _pickMimeType(): string {
const candidates = [
'audio/webm;codecs=opus',
'audio/webm',
'audio/ogg;codecs=opus',
'audio/ogg',
'audio/mp4',
];
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
}
return '';
}
}
export const wasmSttService = new WasmSttService();
@@ -1,92 +0,0 @@
/**
* Web Worker for off-main-thread Whisper transcription.
*
* Receives `{ type: 'load', modelId }` to load a model, then
* `{ type: 'transcribe', audio: Float32Array (transferred buffer), language? }`
* to run inference. Posts progress, results, and errors back.
*/
import { pipeline, env } from '@xenova/transformers';
let transcriber: unknown = null;
self.onmessage = async (e: MessageEvent) => {
const { type } = e.data as { type: string };
if (type === 'load') {
const { modelId } = e.data as { modelId: string };
try {
env.backends.onnx.wasm.numThreads = 1;
const fileDoneBytes = new Map<string, number>();
let totalDone = 0;
let totalEstimate = 0;
transcriber = await pipeline('automatic-speech-recognition', modelId, {
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
if (info.status === 'progress' && info.file) {
const prevDone = fileDoneBytes.get(info.file) ?? 0;
const currentDone = info.loaded ?? 0;
const delta = Math.max(0, currentDone - prevDone);
fileDoneBytes.set(info.file, currentDone);
totalDone += delta;
if (info.total && info.total > totalEstimate) {
totalEstimate = info.total;
}
const effectiveTotal = Math.max(totalEstimate, totalDone);
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
self.postMessage({ type: 'progress', progress: pct });
}
},
});
self.postMessage({ type: 'loaded' });
} catch (err) {
self.postMessage({
type: 'error',
error: err instanceof Error ? err.message : 'Failed to load model',
});
}
} else if (type === 'transcribe') {
if (!transcriber) {
self.postMessage({ type: 'error', error: 'Model not loaded', seq: (e.data as { seq?: number }).seq });
return;
}
const { audio, language, seq } = e.data as { audio: ArrayBuffer; language?: string; seq?: number };
try {
const samples = new Float32Array(audio);
if (samples.length === 0) {
self.postMessage({ type: 'error', error: 'Empty audio received', seq });
return;
}
self.postMessage({ type: 'log', text: `Transcribing ${samples.length} samples (${(samples.length / 16000).toFixed(1)}s)` });
const pipelineFn = transcriber as (
input: Float32Array,
options?: Record<string, unknown>,
) => Promise<{ text: string }>;
const result = await pipelineFn(samples, {
task: 'transcribe',
...(language ? { language } : {}),
});
self.postMessage({
type: 'result',
transcript: (result?.text ?? '').trim(),
seq,
});
} catch (err) {
self.postMessage({
type: 'error',
error: err instanceof Error ? err.message : 'Transcription failed',
seq,
});
}
}
};
+81 -128
View File
@@ -23,10 +23,6 @@ import { getSyncConfig, subscribeToSyncConfigChanges } from "@/sync/sync-refs";
const MODELS_DEV_API_URL = "https://models.dev/api.json";
const MODELS_DEV_PROXY_URL = "/api/openchamber/models-metadata";
const STT_SILENCE_THRESHOLD_DB_MIN = -100;
const STT_SILENCE_THRESHOLD_DB_MAX = 0;
const STT_SILENCE_HOLD_MS_MIN = 250;
const STT_SILENCE_HOLD_MS_MAX = 10000;
const FALLBACK_PROVIDER_ID = "opencode";
const FALLBACK_MODEL_ID = "big-pickle";
@@ -38,18 +34,19 @@ const GIT_UTILITY_PROVIDER_ID = "zen";
const GIT_UTILITY_PREFERRED_MODEL_ID = "big-pickle";
const PROVIDER_CONFIG_REFRESH_CONCURRENCY = 4;
const normalizeSttSilenceThresholdDb = (value: unknown): number | undefined => {
if (typeof value !== 'number' || !Number.isFinite(value)) {
return undefined;
const normalizeSttProvider = (value: unknown): 'local' | 'openai-compatible' | undefined => {
if (value === 'local' || value === 'openai-compatible') {
return value;
}
return Math.max(STT_SILENCE_THRESHOLD_DB_MIN, Math.min(STT_SILENCE_THRESHOLD_DB_MAX, value));
};
const normalizeSttSilenceHoldMs = (value: unknown): number | undefined => {
if (typeof value !== 'number' || !Number.isFinite(value)) {
return undefined;
// Legacy providers: 'server' used an OpenAI-compatible endpoint;
// 'browser' and 'wasm' map to the local default.
if (value === 'server') {
return 'openai-compatible';
}
return Math.max(STT_SILENCE_HOLD_MS_MIN, Math.min(STT_SILENCE_HOLD_MS_MAX, Math.round(value)));
if (value === 'browser' || value === 'wasm') {
return 'local';
}
return undefined;
};
interface OpenChamberDefaults {
@@ -61,13 +58,11 @@ interface OpenChamberDefaults {
defaultFileViewerPreview?: boolean;
zenModel?: string;
messageStreamTransport?: 'auto' | 'ws' | 'sse';
sttProvider?: 'browser' | 'server' | 'wasm';
sttProvider?: 'local' | 'openai-compatible';
sttServerUrl?: string;
wasmSttModel?: string;
sttModel?: string;
sttLocalModel?: string;
sttLanguage?: string;
sttSilenceThresholdDb?: number;
sttSilenceHoldMs?: number;
}
const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
@@ -101,12 +96,11 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
? data.messageStreamTransport
: undefined;
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' || data?.sttProvider === 'wasm' ? data.sttProvider : undefined;
const sttProvider = normalizeSttProvider(data?.sttProvider);
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
const sttLocalModel = typeof data?.sttLocalModel === 'string' ? data.sttLocalModel.trim() : undefined;
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
const sttSilenceThresholdDb = normalizeSttSilenceThresholdDb(data?.sttSilenceThresholdDb);
const sttSilenceHoldMs = normalizeSttSilenceHoldMs(data?.sttSilenceHoldMs);
return finish('runtime-settings', {
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
@@ -120,9 +114,8 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
sttProvider,
sttServerUrl,
sttModel,
sttLocalModel,
sttLanguage,
sttSilenceThresholdDb,
sttSilenceHoldMs,
});
}
} catch {
@@ -149,12 +142,11 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
? data.messageStreamTransport
: undefined;
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' ? data.sttProvider : undefined;
const sttProvider = normalizeSttProvider(data?.sttProvider);
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
const sttLocalModel = typeof data?.sttLocalModel === 'string' ? data.sttLocalModel.trim() : undefined;
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
const sttSilenceThresholdDb = normalizeSttSilenceThresholdDb(data?.sttSilenceThresholdDb);
const sttSilenceHoldMs = normalizeSttSilenceHoldMs(data?.sttSilenceHoldMs);
return finish('settings-route', {
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
@@ -168,9 +160,8 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
sttProvider,
sttServerUrl,
sttModel,
sttLocalModel,
sttLanguage,
sttSilenceThresholdDb,
sttSilenceHoldMs,
});
} catch (error) {
markStartupTrace('config.defaults:error', { error: error instanceof Error ? error.message : String(error) });
@@ -989,33 +980,31 @@ interface ConfigStore {
settingsZenModel: string | undefined;
settingsMessageStreamTransport: 'auto' | 'ws' | 'sse';
// Voice provider preference ('browser', 'openai', 'openai-compatible', or 'say' for macOS)
voiceProvider: 'browser' | 'openai' | 'openai-compatible' | 'say';
setVoiceProvider: (provider: 'browser' | 'openai' | 'openai-compatible' | 'say') => void;
voiceProvider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say';
setVoiceProvider: (provider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say') => void;
// TTS settings
speechRate: number;
speechPitch: number;
speechVolume: number;
sayVoice: string;
browserVoice: string;
localTtsVoiceId: number;
openaiVoice: string;
openaiApiKey: string;
openaiCompatibleUrl: string;
openaiCompatibleApiKey: string;
openaiCompatibleVoice: string;
openaiCompatibleTtsModel: string;
// STT (speech-to-text) settings
sttProvider: 'browser' | 'server' | 'wasm';
// STT (dictation) settings
dictationEnabled: boolean;
sttProvider: 'local' | 'openai-compatible';
sttServerUrl: string;
sttApiKey: string;
sttModel: string;
wasmSttModel: string;
sttLocalModel: string;
sttLanguage: string;
sttSilenceThresholdDb: number;
sttSilenceHoldMs: number;
sttTranscribeOnStop: boolean;
showMessageTTSButtons: boolean;
ttsInputMode: 'sanitized' | 'raw';
voiceModeEnabled: boolean;
// Summarization settings
summarizeMessageTTS: boolean;
summarizeVoiceConversation: boolean;
@@ -1026,24 +1015,22 @@ interface ConfigStore {
setSpeechVolume: (volume: number) => void;
setSayVoice: (voice: string) => void;
setBrowserVoice: (voice: string) => void;
setLocalTtsVoiceId: (voiceId: number) => void;
setOpenaiVoice: (voice: string) => void;
setOpenaiApiKey: (apiKey: string) => void;
setOpenaiCompatibleUrl: (url: string) => void;
setOpenaiCompatibleApiKey: (apiKey: string) => void;
setOpenaiCompatibleVoice: (voice: string) => void;
setOpenaiCompatibleTtsModel: (model: string) => void;
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => void;
setDictationEnabled: (enabled: boolean) => void;
setSttProvider: (provider: 'local' | 'openai-compatible') => void;
setSttServerUrl: (url: string) => void;
setSttApiKey: (apiKey: string) => void;
setSttModel: (model: string) => void;
setWasmSttModel: (model: string) => void;
setSttLocalModel: (model: string) => void;
setSttLanguage: (lang: string) => void;
setSttSilenceThresholdDb: (db: number) => void;
setSttSilenceHoldMs: (ms: number) => void;
setSttTranscribeOnStop: (enabled: boolean) => void;
setShowMessageTTSButtons: (show: boolean) => void;
setTtsInputMode: (mode: 'sanitized' | 'raw') => void;
setVoiceModeEnabled: (enabled: boolean) => void;
setSummarizeMessageTTS: (enabled: boolean) => void;
setSummarizeVoiceConversation: (enabled: boolean) => void;
setSummarizeCharacterThreshold: (threshold: number) => void;
@@ -1136,7 +1123,7 @@ export const useConfigStore = create<ConfigStore>()(
voiceProvider: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('voiceProvider');
if (saved === 'openai' || saved === 'browser' || saved === 'say' || saved === 'openai-compatible') return saved;
if (saved === 'openai' || saved === 'browser' || saved === 'local' || saved === 'say' || saved === 'openai-compatible') return saved;
}
return 'browser';
})(),
@@ -1179,6 +1166,17 @@ export const useConfigStore = create<ConfigStore>()(
}
return 'Samantha';
})(),
// Local (Kokoro) TTS speaker id - load from localStorage or default to 0
localTtsVoiceId: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('localTtsVoiceId');
if (saved !== null) {
const parsed = Number.parseInt(saved, 10);
if (Number.isInteger(parsed) && parsed >= 0) return parsed;
}
}
return 0;
})(),
// Browser voice - load from localStorage or default to empty (auto-select)
browserVoice: (() => {
if (typeof window !== 'undefined') {
@@ -1235,17 +1233,24 @@ export const useConfigStore = create<ConfigStore>()(
}
return 'kokoro';
})(),
// STT provider: 'browser' (Web Speech API), 'server' (OpenAI-compat), 'wasm' (local Whisper)
// Voice input (dictation) master toggle - default enabled
dictationEnabled: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('dictationEnabled');
if (saved === 'false') return false;
}
return true;
})(),
// STT provider: 'local' (server-side sherpa-onnx) or 'openai-compatible'
sttProvider: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('sttProvider');
if (saved === 'browser' || saved === 'server' || saved === 'wasm') return saved;
// Electron/Chromium's Web Speech API requires Google API keys
// not available in Electron, so default to WASM local Whisper.
const electron = (window as unknown as { __OPENCHAMBER_ELECTRON__?: { runtime?: string } }).__OPENCHAMBER_ELECTRON__;
if (electron?.runtime === 'electron') return 'wasm' as const;
if (saved === 'local' || saved === 'openai-compatible') return saved;
// Migrate legacy providers: 'server' used an OpenAI-compatible
// endpoint; 'browser' and 'wasm' map to the local default.
if (saved === 'server') return 'openai-compatible' as const;
}
return 'browser' as const;
return 'local' as const;
})(),
sttServerUrl: (() => {
if (typeof window !== 'undefined') {
@@ -1268,12 +1273,12 @@ export const useConfigStore = create<ConfigStore>()(
}
return 'deepdml/faster-whisper-large-v3-turbo-ct2';
})(),
wasmSttModel: (() => {
sttLocalModel: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('wasmSttModel');
const saved = localStorage.getItem('sttLocalModel');
if (saved) return saved;
}
return 'Xenova/whisper-base.en';
return 'parakeet-tdt-0.6b-v2-int8';
})(),
sttLanguage: (() => {
if (typeof window !== 'undefined') {
@@ -1282,33 +1287,6 @@ export const useConfigStore = create<ConfigStore>()(
}
return '';
})(),
sttSilenceThresholdDb: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('sttSilenceThresholdDb');
if (saved) {
const parsed = parseFloat(saved);
if (!isNaN(parsed)) return parsed;
}
}
return -45;
})(),
sttSilenceHoldMs: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('sttSilenceHoldMs');
if (saved) {
const parsed = parseInt(saved, 10);
if (!isNaN(parsed)) return parsed;
}
}
return 1500;
})(),
sttTranscribeOnStop: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('sttTranscribeOnStop');
if (saved === 'true') return true;
}
return false;
})(),
// Show TTS buttons on messages - disabled by default until user enables it
showMessageTTSButtons: (() => {
if (typeof window !== 'undefined') {
@@ -1324,14 +1302,6 @@ export const useConfigStore = create<ConfigStore>()(
}
return 'sanitized' as const;
})(),
// Voice mode enabled - load from localStorage or default to false
voiceModeEnabled: (() => {
if (typeof window !== 'undefined') {
const saved = localStorage.getItem('voiceModeEnabled');
if (saved === 'true') return true;
}
return false;
})(),
// Summarization settings
summarizeMessageTTS: (() => {
if (typeof window !== 'undefined') {
@@ -2060,9 +2030,8 @@ export const useConfigStore = create<ConfigStore>()(
sttProvider: openChamberDefaults.sttProvider ?? state.sttProvider,
sttServerUrl: openChamberDefaults.sttServerUrl ?? state.sttServerUrl,
sttModel: openChamberDefaults.sttModel ?? state.sttModel,
sttLocalModel: openChamberDefaults.sttLocalModel ?? state.sttLocalModel,
sttLanguage: openChamberDefaults.sttLanguage ?? state.sttLanguage,
sttSilenceThresholdDb: openChamberDefaults.sttSilenceThresholdDb ?? state.sttSilenceThresholdDb,
sttSilenceHoldMs: openChamberDefaults.sttSilenceHoldMs ?? state.sttSilenceHoldMs,
directoryScoped: {
...state.directoryScoped,
[directoryKey]: nextSnapshot,
@@ -2800,7 +2769,7 @@ export const useConfigStore = create<ConfigStore>()(
});
},
setVoiceProvider: (provider: 'browser' | 'openai' | 'openai-compatible' | 'say') => {
setVoiceProvider: (provider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say') => {
set({ voiceProvider: provider });
if (typeof window !== 'undefined') {
localStorage.setItem('voiceProvider', provider);
@@ -2838,6 +2807,13 @@ export const useConfigStore = create<ConfigStore>()(
}
},
setLocalTtsVoiceId: (voiceId: number) => {
set({ localTtsVoiceId: voiceId });
if (typeof window !== 'undefined') {
localStorage.setItem('localTtsVoiceId', String(voiceId));
}
},
setBrowserVoice: (voice: string) => {
set({ browserVoice: voice });
if (typeof window !== 'undefined') {
@@ -2887,7 +2863,15 @@ export const useConfigStore = create<ConfigStore>()(
}
},
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => {
setDictationEnabled: (enabled: boolean) => {
set({ dictationEnabled: enabled });
if (typeof window !== 'undefined') {
localStorage.setItem('dictationEnabled', String(enabled));
}
updateDesktopSettings({ dictationEnabled: enabled }).catch(() => {});
},
setSttProvider: (provider: 'local' | 'openai-compatible') => {
set({ sttProvider: provider });
if (typeof window !== 'undefined') {
localStorage.setItem('sttProvider', provider);
@@ -2918,12 +2902,12 @@ export const useConfigStore = create<ConfigStore>()(
updateDesktopSettings({ sttModel: model }).catch(() => {});
},
setWasmSttModel: (model: string) => {
set({ wasmSttModel: model });
setSttLocalModel: (model: string) => {
set({ sttLocalModel: model });
if (typeof window !== 'undefined') {
localStorage.setItem('wasmSttModel', model);
localStorage.setItem('sttLocalModel', model);
}
updateDesktopSettings({ wasmSttModel: model }).catch(() => {});
updateDesktopSettings({ sttLocalModel: model }).catch(() => {});
},
setSttLanguage: (lang: string) => {
@@ -2934,30 +2918,6 @@ export const useConfigStore = create<ConfigStore>()(
updateDesktopSettings({ sttLanguage: lang }).catch(() => {});
},
setSttSilenceThresholdDb: (db: number) => {
set({ sttSilenceThresholdDb: db });
if (typeof window !== 'undefined') {
localStorage.setItem('sttSilenceThresholdDb', String(db));
}
updateDesktopSettings({ sttSilenceThresholdDb: db }).catch(() => {});
},
setSttSilenceHoldMs: (ms: number) => {
set({ sttSilenceHoldMs: ms });
if (typeof window !== 'undefined') {
localStorage.setItem('sttSilenceHoldMs', String(ms));
}
updateDesktopSettings({ sttSilenceHoldMs: ms }).catch(() => {});
},
setSttTranscribeOnStop: (enabled: boolean) => {
set({ sttTranscribeOnStop: enabled });
if (typeof window !== 'undefined') {
localStorage.setItem('sttTranscribeOnStop', String(enabled));
}
updateDesktopSettings({ sttTranscribeOnStop: enabled }).catch(() => {});
},
setShowMessageTTSButtons: (show: boolean) => {
set({ showMessageTTSButtons: show });
if (typeof window !== 'undefined') {
@@ -2972,13 +2932,6 @@ export const useConfigStore = create<ConfigStore>()(
}
},
setVoiceModeEnabled: (enabled: boolean) => {
set({ voiceModeEnabled: enabled });
if (typeof window !== 'undefined') {
localStorage.setItem('voiceModeEnabled', String(enabled));
}
},
setSummarizeMessageTTS: (enabled: boolean) => {
set({ summarizeMessageTTS: enabled });
if (typeof window !== 'undefined') {
-1
View File
@@ -190,7 +190,6 @@ function notifyMessageSent(sessionId: string): void {
export type { SyntheticContextPart } from "./input-store"
export type { SessionMemoryState } from "./viewport-store"
export type { VoiceStatus, VoiceMode } from "./voice-store"
export type NewSessionDraftState = {
open: boolean
File diff suppressed because it is too large Load Diff
-7
View File
@@ -1,7 +0,0 @@
/**
* Voice Store voice connection and activity state.
* Extracted from session-ui-store for subscription isolation.
*/
export type VoiceStatus = "disconnected" | "connecting" | "connected" | "error"
export type VoiceMode = "idle" | "speaking" | "listening"