feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)
Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
This commit is contained in:
committed by
GitHub
parent
3f5151d424
commit
de1b85ac56
@@ -32,7 +32,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "22"
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
@@ -206,7 +206,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
registry-url: 'https://registry.npmjs.org'
|
registry-url: 'https://registry.npmjs.org'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
@@ -141,7 +141,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
@@ -288,7 +288,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
@@ -365,7 +365,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Download per-arch latest-mac.yml
|
- name: Download per-arch latest-mac.yml
|
||||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ jobs:
|
|||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '22'
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install --frozen-lockfile
|
run: bun install --frozen-lockfile
|
||||||
|
|||||||
@@ -67,3 +67,4 @@ data/
|
|||||||
workspaces/
|
workspaces/
|
||||||
*.pid
|
*.pid
|
||||||
.worktrees/
|
.worktrees/
|
||||||
|
test-results/
|
||||||
|
|||||||
@@ -283,6 +283,7 @@
|
|||||||
"openai": "^4.79.0",
|
"openai": "^4.79.0",
|
||||||
"qrcode-terminal": "^0.12.0",
|
"qrcode-terminal": "^0.12.0",
|
||||||
"reflect-metadata": "^0.2.2",
|
"reflect-metadata": "^0.2.2",
|
||||||
|
"sherpa-onnx-node": "1.12.28",
|
||||||
"simple-git": "^3.28.0",
|
"simple-git": "^3.28.0",
|
||||||
"web-push": "^3.6.7",
|
"web-push": "^3.6.7",
|
||||||
"ws": "^8.18.3",
|
"ws": "^8.18.3",
|
||||||
@@ -2980,6 +2981,20 @@
|
|||||||
|
|
||||||
"shell-quote": ["shell-quote@1.8.3", "", {}, "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw=="],
|
"shell-quote": ["shell-quote@1.8.3", "", {}, "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-darwin-arm64": ["sherpa-onnx-darwin-arm64@1.13.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9x86Cbf+BDFONdtCPM3cnjvtAW0ER8tMaHK5pVfz+SHPt8GeuwRXaiR/BzcByFBUyxCgmceO09/WMZOCi44P/g=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-darwin-x64": ["sherpa-onnx-darwin-x64@1.13.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-TVQ35g7JIpDPB1lUDdcog+JtI0cI45ZzOnvHXm0DtWs/dgxnJXtWMY3uLRtBbLnysV9j5ljffwZ1IX9VDHsCzQ=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-linux-arm64": ["sherpa-onnx-linux-arm64@1.13.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-uDtZkkoP6QQ/3DHOscCpEZ2WpaiHUQsDpbyYaHURrJ7DbsjqGnS6G8l+R589Ro5Bf282QElzBy3okwxXbt3Kxw=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-linux-x64": ["sherpa-onnx-linux-x64@1.13.3", "", { "os": "linux", "cpu": "x64" }, "sha512-OFVK0GYwKwKNsjxbPmfcLQm/dfA0IwAoiIQJ96s+eFYcDqhlapcY06ocdb7SNluGBcM7xgU5jEW2QXBkMIOEvQ=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-node": ["sherpa-onnx-node@1.12.28", "", { "optionalDependencies": { "sherpa-onnx-darwin-arm64": "^1.12.28", "sherpa-onnx-darwin-x64": "^1.12.28", "sherpa-onnx-linux-arm64": "^1.12.28", "sherpa-onnx-linux-x64": "^1.12.28", "sherpa-onnx-win-ia32": "^1.12.28", "sherpa-onnx-win-x64": "^1.12.28" } }, "sha512-EHSB3EG6hKyXaTNh6GU/bwh6i3dncCH6ZCU2mScNzkxRbVStZ7QmNj0Oo4E9XrGGo9jX9pKg9MPHEPyjdK+ApA=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-win-ia32": ["sherpa-onnx-win-ia32@1.13.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-VDZh1M7Ccx/bkP3WwBCFoJzwAwq+b5nR1KRkYRz5p1w5bfhzfa3ACBGr7vpUt5AGUge4qSLe0MSKXyKtSmy1uA=="],
|
||||||
|
|
||||||
|
"sherpa-onnx-win-x64": ["sherpa-onnx-win-x64@1.13.3", "", { "os": "win32", "cpu": "x64" }, "sha512-ZQzcSmFvZK4jzmtWckqxocDUuEjYnBV2MHrDD21HPTeUMfGdE9yfvuSPpesIVfdzKbQzIQY42RAcfZEGWu0FbQ=="],
|
||||||
|
|
||||||
"shiki": ["shiki@3.23.0", "", { "dependencies": { "@shikijs/core": "3.23.0", "@shikijs/engine-javascript": "3.23.0", "@shikijs/engine-oniguruma": "3.23.0", "@shikijs/langs": "3.23.0", "@shikijs/themes": "3.23.0", "@shikijs/types": "3.23.0", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA=="],
|
"shiki": ["shiki@3.23.0", "", { "dependencies": { "@shikijs/core": "3.23.0", "@shikijs/engine-javascript": "3.23.0", "@shikijs/engine-oniguruma": "3.23.0", "@shikijs/langs": "3.23.0", "@shikijs/themes": "3.23.0", "@shikijs/types": "3.23.0", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA=="],
|
||||||
|
|
||||||
"side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
|
"side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
|
||||||
|
|||||||
@@ -53,4 +53,10 @@
|
|||||||
<!-- Android 13+ requires this declared for FCM push notifications to be shown; the runtime
|
<!-- Android 13+ requires this declared for FCM push notifications to be shown; the runtime
|
||||||
prompt is requested by @capacitor/push-notifications on register. -->
|
prompt is requested by @capacitor/push-notifications on register. -->
|
||||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
||||||
|
<!-- Microphone: voice dictation via WebView getUserMedia. Capacitor's
|
||||||
|
BridgeWebChromeClient forwards the WebView AUDIO_CAPTURE permission
|
||||||
|
request to the Android runtime prompt when this is declared. -->
|
||||||
|
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||||
|
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
|
||||||
|
<uses-feature android:name="android.hardware.microphone" android:required="false" />
|
||||||
</manifest>
|
</manifest>
|
||||||
|
|||||||
@@ -35,6 +35,8 @@
|
|||||||
<string>OpenChamber connects to OpenChamber servers on your local network.</string>
|
<string>OpenChamber connects to OpenChamber servers on your local network.</string>
|
||||||
<key>NSCameraUsageDescription</key>
|
<key>NSCameraUsageDescription</key>
|
||||||
<string>OpenChamber uses the camera to scan a server's pairing QR code.</string>
|
<string>OpenChamber uses the camera to scan a server's pairing QR code.</string>
|
||||||
|
<key>NSMicrophoneUsageDescription</key>
|
||||||
|
<string>OpenChamber uses the microphone for voice dictation in the chat composer.</string>
|
||||||
<key>CFBundleURLTypes</key>
|
<key>CFBundleURLTypes</key>
|
||||||
<array>
|
<array>
|
||||||
<dict>
|
<dict>
|
||||||
|
|||||||
@@ -44,7 +44,6 @@ import { ConfigUpdateOverlay } from '@/components/ui/ConfigUpdateOverlay';
|
|||||||
import { AboutDialog } from '@/components/ui/AboutDialog';
|
import { AboutDialog } from '@/components/ui/AboutDialog';
|
||||||
import { RuntimeAPIProvider } from '@/contexts/RuntimeAPIProvider';
|
import { RuntimeAPIProvider } from '@/contexts/RuntimeAPIProvider';
|
||||||
import { registerRuntimeAPIs } from '@/contexts/runtimeAPIRegistry';
|
import { registerRuntimeAPIs } from '@/contexts/runtimeAPIRegistry';
|
||||||
import { VoiceProvider } from '@/components/voice';
|
|
||||||
import { useUIStore } from '@/stores/useUIStore';
|
import { useUIStore } from '@/stores/useUIStore';
|
||||||
import { useGitHubAuthStore } from '@/stores/useGitHubAuthStore';
|
import { useGitHubAuthStore } from '@/stores/useGitHubAuthStore';
|
||||||
import { useFeatureFlagsStore } from '@/stores/useFeatureFlagsStore';
|
import { useFeatureFlagsStore } from '@/stores/useFeatureFlagsStore';
|
||||||
@@ -928,8 +927,8 @@ function App({ apis }: AppProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Always mount the full provider tree to avoid remounts when isInitialized
|
// Always mount the full provider tree to avoid remounts when isInitialized
|
||||||
// flips from false → true. FireworksProvider and VoiceProvider are lightweight
|
// flips from false → true. FireworksProvider is a lightweight shell; its
|
||||||
// shells; their heavy children are only activated when actually needed.
|
// heavy children are only activated when actually needed.
|
||||||
const isBootShell = !isInitialized && !isDesktopRuntime;
|
const isBootShell = !isInitialized && !isDesktopRuntime;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -937,7 +936,6 @@ function App({ apis }: AppProps) {
|
|||||||
<SyncProvider key={runtimeEndpointEpoch} sdk={opencodeClient.getSdkClient()} directory={currentDirectory || ''}>
|
<SyncProvider key={runtimeEndpointEpoch} sdk={opencodeClient.getSdkClient()} directory={currentDirectory || ''}>
|
||||||
<RuntimeAPIProvider apis={apis}>
|
<RuntimeAPIProvider apis={apis}>
|
||||||
<FireworksProvider>
|
<FireworksProvider>
|
||||||
<VoiceProvider>
|
|
||||||
<TooltipProvider delayDuration={300} skipDelayDuration={150}>
|
<TooltipProvider delayDuration={300} skipDelayDuration={150}>
|
||||||
<div className={isDesktopRuntime ? 'h-full text-foreground bg-transparent' : 'h-full text-foreground bg-background'}>
|
<div className={isDesktopRuntime ? 'h-full text-foreground bg-transparent' : 'h-full text-foreground bg-background'}>
|
||||||
<SyncAppEffects embeddedBackgroundWorkEnabled={embeddedBackgroundWorkEnabled} />
|
<SyncAppEffects embeddedBackgroundWorkEnabled={embeddedBackgroundWorkEnabled} />
|
||||||
@@ -955,7 +953,6 @@ function App({ apis }: AppProps) {
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</TooltipProvider>
|
</TooltipProvider>
|
||||||
</VoiceProvider>
|
|
||||||
</FireworksProvider>
|
</FireworksProvider>
|
||||||
</RuntimeAPIProvider>
|
</RuntimeAPIProvider>
|
||||||
</SyncProvider>
|
</SyncProvider>
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import React from 'react';
|
import React from 'react';
|
||||||
import { Textarea } from '@/components/ui/textarea';
|
import { Textarea } from '@/components/ui/textarea';
|
||||||
import { BrowserVoiceButton } from '@/components/voice';
|
import { ComposerDictation } from '@/components/dictation/ComposerDictation';
|
||||||
// sessionStore removed — currentSessionId comes from useSessionUIStore
|
// sessionStore removed — currentSessionId comes from useSessionUIStore
|
||||||
import { useConfigStore } from '@/stores/useConfigStore';
|
import { useConfigStore } from '@/stores/useConfigStore';
|
||||||
import { useUIStore } from '@/stores/useUIStore';
|
import { useUIStore } from '@/stores/useUIStore';
|
||||||
@@ -381,7 +381,7 @@ const getProjectIconColor = (projectColor?: string | null): string | undefined =
|
|||||||
};
|
};
|
||||||
|
|
||||||
const MemoModelControls = React.memo(ModelControls);
|
const MemoModelControls = React.memo(ModelControls);
|
||||||
const MemoBrowserVoiceButton = React.memo(BrowserVoiceButton);
|
const MemoComposerDictation = React.memo(ComposerDictation);
|
||||||
const MemoMobileAgentButton = React.memo(MobileAgentButton);
|
const MemoMobileAgentButton = React.memo(MobileAgentButton);
|
||||||
const MemoMobileModelButton = React.memo(MobileModelButton);
|
const MemoMobileModelButton = React.memo(MobileModelButton);
|
||||||
const MemoStatusRow = React.memo(StatusRow);
|
const MemoStatusRow = React.memo(StatusRow);
|
||||||
@@ -2318,6 +2318,33 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
|||||||
void handleSubmitRef.current();
|
void handleSubmitRef.current();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
// Dictation: insert the transcript inline; optionally submit immediately.
|
||||||
|
// getCurrentInputSnapshot reads textareaRef.current.value first, so setting
|
||||||
|
// it synchronously lets handleSubmit pick up the text in the same tick.
|
||||||
|
const handleDictationInsert = React.useCallback((text: string) => {
|
||||||
|
setMessage((prev) => {
|
||||||
|
const next = appendInlineText(prev, text);
|
||||||
|
const textarea = textareaRef.current;
|
||||||
|
if (textarea) {
|
||||||
|
textarea.value = next;
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
setTimeout(() => {
|
||||||
|
textareaRef.current?.focus();
|
||||||
|
}, 0);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleDictationInsertAndSend = React.useCallback((text: string) => {
|
||||||
|
const textarea = textareaRef.current;
|
||||||
|
const next = appendInlineText(textarea?.value ?? messageRef.current, text);
|
||||||
|
if (textarea) {
|
||||||
|
textarea.value = next;
|
||||||
|
}
|
||||||
|
setMessage(next);
|
||||||
|
void handleSubmitRef.current();
|
||||||
|
}, []);
|
||||||
|
|
||||||
// Preset chips rendered outside this component (e.g. under the welcome
|
// Preset chips rendered outside this component (e.g. under the welcome
|
||||||
// message on narrow surfaces) request a submit via the input store; consume
|
// message on narrow surfaces) request a submit via the input store; consume
|
||||||
// it here so it routes through the same command-aware submit path.
|
// it here so it routes through the same command-aware submit path.
|
||||||
@@ -4420,6 +4447,9 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
|||||||
: undefined}
|
: undefined}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
{/* Positioning context for the dictation overlay: covers the
|
||||||
|
text area + footer exactly, excluding MobileSessionStatusBar. */}
|
||||||
|
<div className={cn('relative flex flex-col', isDesktopExpanded && 'flex-1 min-h-0')}>
|
||||||
<div className={cn("overflow-hidden", isDesktopExpanded && 'flex flex-1 min-h-0 flex-col')}>
|
<div className={cn("overflow-hidden", isDesktopExpanded && 'flex flex-1 min-h-0 flex-col')}>
|
||||||
<div className="flex items-center gap-1 px-3 pt-1 flex-wrap relative z-10">
|
<div className="flex items-center gap-1 px-3 pt-1 flex-wrap relative z-10">
|
||||||
<AttachedVSCodeFileChips onShowPopup={handleShowAttachmentPreview} />
|
<AttachedVSCodeFileChips onShowPopup={handleShowAttachmentPreview} />
|
||||||
@@ -4559,7 +4589,16 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<div className="flex items-center gap-x-1 flex-shrink-0">
|
<div className="flex items-center gap-x-1 flex-shrink-0">
|
||||||
<MemoBrowserVoiceButton />
|
<MemoComposerDictation
|
||||||
|
radius={chatInputRadius}
|
||||||
|
isMobile={isMobile}
|
||||||
|
footerIconButtonClass={footerIconButtonClass}
|
||||||
|
footerPaddingClass={footerPaddingClass}
|
||||||
|
iconSizeClass={iconSizeClass}
|
||||||
|
sendIconSizeClass={sendIconSizeClass}
|
||||||
|
onInsert={handleDictationInsert}
|
||||||
|
onInsertAndSend={handleDictationInsertAndSend}
|
||||||
|
/>
|
||||||
<ComposerActionButtons
|
<ComposerActionButtons
|
||||||
isMobile={isMobile}
|
isMobile={isMobile}
|
||||||
footerIconButtonClass={footerIconButtonClass}
|
footerIconButtonClass={footerIconButtonClass}
|
||||||
@@ -4614,7 +4653,16 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
|||||||
</div>
|
</div>
|
||||||
<div className={cn('flex items-center flex-1 justify-end', footerGapClass, 'md:gap-x-3')}>
|
<div className={cn('flex items-center flex-1 justify-end', footerGapClass, 'md:gap-x-3')}>
|
||||||
<MemoModelControls className={cn('flex-1 min-w-0 justify-end')} />
|
<MemoModelControls className={cn('flex-1 min-w-0 justify-end')} />
|
||||||
<MemoBrowserVoiceButton />
|
<MemoComposerDictation
|
||||||
|
radius={chatInputRadius}
|
||||||
|
isMobile={isMobile}
|
||||||
|
footerIconButtonClass={footerIconButtonClass}
|
||||||
|
footerPaddingClass={footerPaddingClass}
|
||||||
|
iconSizeClass={iconSizeClass}
|
||||||
|
sendIconSizeClass={sendIconSizeClass}
|
||||||
|
onInsert={handleDictationInsert}
|
||||||
|
onInsertAndSend={handleDictationInsertAndSend}
|
||||||
|
/>
|
||||||
<ComposerActionButtons
|
<ComposerActionButtons
|
||||||
isMobile={isMobile}
|
isMobile={isMobile}
|
||||||
footerIconButtonClass={footerIconButtonClass}
|
footerIconButtonClass={footerIconButtonClass}
|
||||||
@@ -4633,6 +4681,7 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* Mobile session panel: slide-up overlay toggled by MobileSessionPanelTrigger. */}
|
{/* Mobile session panel: slide-up overlay toggled by MobileSessionPanelTrigger. */}
|
||||||
{isMobile && <MobileSessionStatusBar />}
|
{isMobile && <MobileSessionStatusBar />}
|
||||||
|
|||||||
@@ -0,0 +1,439 @@
|
|||||||
|
/**
|
||||||
|
* Composer dictation controls: a mic button for the composer footer plus a
|
||||||
|
* full-composer overlay while dictation is active (recording, transcribing,
|
||||||
|
* or failed). The overlay mirrors the composer's own layout — the transcript
|
||||||
|
* area uses the same paddings/typography as the textarea and the action row
|
||||||
|
* reuses the footer icon-button styling — so toggling dictation causes no
|
||||||
|
* vertical shift.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import React from 'react';
|
||||||
|
|
||||||
|
import { Icon } from '@/components/icon/Icon';
|
||||||
|
import { useI18n } from '@/lib/i18n';
|
||||||
|
import { useThemeSystem } from '@/contexts/useThemeSystem';
|
||||||
|
import { cn } from '@/lib/utils';
|
||||||
|
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||||
|
import { useDictation } from '@/hooks/useDictation';
|
||||||
|
import { isDictationCaptureSupported } from '@/lib/dictation/use-dictation-audio-source';
|
||||||
|
import { isVSCodeRuntime } from '@/lib/desktop';
|
||||||
|
import { useConfigStore } from '@/stores/useConfigStore';
|
||||||
|
import { useUIStore } from '@/stores/useUIStore';
|
||||||
|
import { formatShortcutForDisplay, getEffectiveShortcutCombo } from '@/lib/shortcuts';
|
||||||
|
|
||||||
|
interface ComposerDictationProps {
|
||||||
|
radius?: number | string;
|
||||||
|
isMobile: boolean;
|
||||||
|
footerIconButtonClass: string;
|
||||||
|
footerPaddingClass: string;
|
||||||
|
iconSizeClass: string;
|
||||||
|
sendIconSizeClass: string;
|
||||||
|
disabled?: boolean;
|
||||||
|
onInsert: (text: string) => void;
|
||||||
|
onInsertAndSend: (text: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const formatDuration = (seconds: number): string => {
|
||||||
|
const mins = Math.floor(seconds / 60);
|
||||||
|
const secs = seconds % 60;
|
||||||
|
return `${mins}:${String(secs).padStart(2, '0')}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const VolumeMeter: React.FC<{ volume: number }> = ({ volume }) => {
|
||||||
|
const { currentTheme } = useThemeSystem();
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="h-1.5 w-16 flex-shrink-0 overflow-hidden rounded-full"
|
||||||
|
style={{ backgroundColor: currentTheme.colors.interactive.border }}
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className="h-full rounded-full transition-[width] duration-75"
|
||||||
|
style={{
|
||||||
|
width: `${Math.round(Math.min(1, volume) * 100)}%`,
|
||||||
|
backgroundColor: currentTheme.colors.primary.base,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Polls the dictation status route while the local model is downloading and
|
||||||
|
* returns the download percent (null while unknown / not downloading).
|
||||||
|
*/
|
||||||
|
const useModelDownloadProgress = (active: boolean): number | null => {
|
||||||
|
const sttLocalModel = useConfigStore((state) => state.sttLocalModel);
|
||||||
|
const [percent, setPercent] = React.useState<number | null>(null);
|
||||||
|
|
||||||
|
React.useEffect(() => {
|
||||||
|
if (!active) {
|
||||||
|
setPercent(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
const poll = async () => {
|
||||||
|
try {
|
||||||
|
const response = await runtimeFetch('/api/dictation/status', {
|
||||||
|
query: { provider: 'local', localModel: sttLocalModel },
|
||||||
|
});
|
||||||
|
if (!response.ok || cancelled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const data = await response.json();
|
||||||
|
const model = Array.isArray(data?.models)
|
||||||
|
? data.models.find((m: { id: string }) => m.id === sttLocalModel)
|
||||||
|
: null;
|
||||||
|
if (!cancelled) {
|
||||||
|
setPercent(typeof model?.downloadProgress === 'number' ? model.downloadProgress : null);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Display-only; keep the previous value.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
void poll();
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
void poll();
|
||||||
|
}, 2000);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
clearInterval(interval);
|
||||||
|
};
|
||||||
|
}, [active, sttLocalModel]);
|
||||||
|
|
||||||
|
return active ? percent : null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||||
|
radius,
|
||||||
|
isMobile,
|
||||||
|
footerIconButtonClass,
|
||||||
|
footerPaddingClass,
|
||||||
|
iconSizeClass,
|
||||||
|
sendIconSizeClass,
|
||||||
|
disabled,
|
||||||
|
onInsert,
|
||||||
|
onInsertAndSend,
|
||||||
|
}) => {
|
||||||
|
const { t } = useI18n();
|
||||||
|
const { currentTheme } = useThemeSystem();
|
||||||
|
const dictationEnabled = useConfigStore((state) => state.dictationEnabled);
|
||||||
|
const shortcutOverrides = useUIStore((state) => state.shortcutOverrides);
|
||||||
|
const dictationShortcut = formatShortcutForDisplay(getEffectiveShortcutCombo('toggle_dictation', shortcutOverrides));
|
||||||
|
// The dictation server (WebSocket + STT worker) lives in the OpenChamber
|
||||||
|
// web server; the VS Code bridge has no server process for it.
|
||||||
|
const [supported] = React.useState(() => !isVSCodeRuntime() && isDictationCaptureSupported());
|
||||||
|
|
||||||
|
const pendingActionRef = React.useRef<'insert' | 'send' | null>(null);
|
||||||
|
const onInsertRef = React.useRef(onInsert);
|
||||||
|
const onInsertAndSendRef = React.useRef(onInsertAndSend);
|
||||||
|
React.useEffect(() => {
|
||||||
|
onInsertRef.current = onInsert;
|
||||||
|
onInsertAndSendRef.current = onInsertAndSend;
|
||||||
|
}, [onInsert, onInsertAndSend]);
|
||||||
|
|
||||||
|
const dictation = useDictation({
|
||||||
|
onTranscript: (text) => {
|
||||||
|
const action = pendingActionRef.current;
|
||||||
|
pendingActionRef.current = null;
|
||||||
|
if (action === 'send') {
|
||||||
|
onInsertAndSendRef.current(text);
|
||||||
|
} else {
|
||||||
|
onInsertRef.current(text);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const {
|
||||||
|
status,
|
||||||
|
partialTranscript,
|
||||||
|
volume,
|
||||||
|
duration,
|
||||||
|
error,
|
||||||
|
errorReason,
|
||||||
|
startDictation,
|
||||||
|
confirmDictation,
|
||||||
|
cancelDictation,
|
||||||
|
retryFailedDictation,
|
||||||
|
acceptPartialTranscript,
|
||||||
|
discardFailedDictation,
|
||||||
|
} = dictation;
|
||||||
|
|
||||||
|
const isModelDownloading = status === 'recording' && errorReason === 'model_download_in_progress';
|
||||||
|
const downloadPercent = useModelDownloadProgress(isModelDownloading);
|
||||||
|
|
||||||
|
const statusRef = React.useRef(status);
|
||||||
|
React.useEffect(() => {
|
||||||
|
statusRef.current = status;
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
|
// Keyboard shortcut (toggle_dictation): idle -> start recording,
|
||||||
|
// recording -> confirm and insert. Dispatched by useKeyboardShortcuts.
|
||||||
|
React.useEffect(() => {
|
||||||
|
const onToggle = () => {
|
||||||
|
if (statusRef.current === 'idle') {
|
||||||
|
void startDictation();
|
||||||
|
} else if (statusRef.current === 'recording') {
|
||||||
|
pendingActionRef.current = 'insert';
|
||||||
|
void confirmDictation();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
window.addEventListener('openchamber:dictation-toggle', onToggle);
|
||||||
|
return () => window.removeEventListener('openchamber:dictation-toggle', onToggle);
|
||||||
|
}, [startDictation, confirmDictation]);
|
||||||
|
|
||||||
|
// While recording: Enter confirms (insert), Escape cancels. Capture-phase
|
||||||
|
// so the composer's own Enter-to-send never fires underneath the overlay.
|
||||||
|
React.useEffect(() => {
|
||||||
|
if (status !== 'recording') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const onKeyDown = (event: KeyboardEvent) => {
|
||||||
|
if (event.isComposing) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.key === 'Enter' && !event.shiftKey && !event.metaKey && !event.ctrlKey && !event.altKey) {
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
pendingActionRef.current = 'insert';
|
||||||
|
void confirmDictation();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.key === 'Escape') {
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
void cancelDictation();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
window.addEventListener('keydown', onKeyDown, { capture: true });
|
||||||
|
return () => window.removeEventListener('keydown', onKeyDown, { capture: true });
|
||||||
|
}, [status, confirmDictation, cancelDictation]);
|
||||||
|
|
||||||
|
// Pixel-parity with the composer: the real footer row is taller than our
|
||||||
|
// buttons (its height comes from the tallest control, e.g. the model
|
||||||
|
// picker), so measure it — it stays mounted underneath the overlay — and
|
||||||
|
// give our action row the same height so the icons line up exactly.
|
||||||
|
const overlayRef = React.useRef<HTMLDivElement | null>(null);
|
||||||
|
const [footerHeight, setFooterHeight] = React.useState<number | null>(null);
|
||||||
|
const isActiveStatus = status !== 'idle';
|
||||||
|
React.useLayoutEffect(() => {
|
||||||
|
if (!isActiveStatus) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// The overlay is rendered inside the composer footer itself, so the
|
||||||
|
// real footer is an ancestor, not a sibling.
|
||||||
|
const realFooter = overlayRef.current?.closest<HTMLElement>('[data-chat-input-footer="true"]');
|
||||||
|
if (realFooter && realFooter.offsetHeight > 0) {
|
||||||
|
setFooterHeight(realFooter.offsetHeight);
|
||||||
|
}
|
||||||
|
}, [isActiveStatus]);
|
||||||
|
|
||||||
|
if (!supported || !dictationEnabled) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const isActive = status !== 'idle';
|
||||||
|
|
||||||
|
const confirmWith = (action: 'insert' | 'send') => {
|
||||||
|
pendingActionRef.current = action;
|
||||||
|
void confirmDictation();
|
||||||
|
};
|
||||||
|
|
||||||
|
const retry = () => {
|
||||||
|
pendingActionRef.current = 'insert';
|
||||||
|
void retryFailedDictation();
|
||||||
|
};
|
||||||
|
|
||||||
|
const placeholderText = (() => {
|
||||||
|
if (status === 'failed') {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
if (status === 'uploading') {
|
||||||
|
return t('chat.dictation.processing');
|
||||||
|
}
|
||||||
|
if (isModelDownloading) {
|
||||||
|
return downloadPercent !== null
|
||||||
|
? t('chat.dictation.downloadingModelProgress', { percent: String(downloadPercent) })
|
||||||
|
: t('chat.dictation.downloadingModel');
|
||||||
|
}
|
||||||
|
return t('chat.dictation.listening');
|
||||||
|
})();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={footerIconButtonClass}
|
||||||
|
onClick={() => {
|
||||||
|
void startDictation();
|
||||||
|
}}
|
||||||
|
disabled={disabled || isActive}
|
||||||
|
title={dictationShortcut ? `${t('chat.dictation.start')} (${dictationShortcut})` : t('chat.dictation.start')}
|
||||||
|
aria-label={t('chat.dictation.start')}
|
||||||
|
>
|
||||||
|
<Icon name="mic" className={cn(iconSizeClass, 'text-current')} />
|
||||||
|
</button>
|
||||||
|
{isActive ? (
|
||||||
|
<div
|
||||||
|
ref={overlayRef}
|
||||||
|
// overflow-x/y split on purpose: mobile.css rewrites the
|
||||||
|
// shorthand `.overflow-hidden` to overflow-y:auto on touch
|
||||||
|
// devices, which painted a phantom scrollbar on Android.
|
||||||
|
className="absolute inset-0 z-50 flex flex-col overflow-x-hidden overflow-y-hidden"
|
||||||
|
style={{
|
||||||
|
borderRadius: radius,
|
||||||
|
// Must match the composer box background exactly so the
|
||||||
|
// overlay reads as the same surface, not a layer on top.
|
||||||
|
backgroundColor: currentTheme.colors.surface.subtle,
|
||||||
|
}}
|
||||||
|
role="dialog"
|
||||||
|
aria-label={t('chat.dictation.overlayAria')}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className={cn(
|
||||||
|
// Text paddings match the composer textarea, plus the
|
||||||
|
// 4px (pt-1) attachment-chips row that always renders
|
||||||
|
// above it: desktop 16+4px, mobile 10+4px from the top.
|
||||||
|
// min-h-0 (not a fixed min height): the mobile composer
|
||||||
|
// is shorter than 52px of text area + footer, and a
|
||||||
|
// fixed min pushed the action row 4px below the real
|
||||||
|
// footer. The area must shrink to whatever space the
|
||||||
|
// underlying composer actually has.
|
||||||
|
'flex-1 min-h-0 overflow-y-auto px-3',
|
||||||
|
isMobile ? 'pt-3.5 pb-2.5' : 'pt-5 pb-2',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{partialTranscript ? (
|
||||||
|
<p className="typography-markdown md:typography-ui-label whitespace-pre-wrap" style={{ color: currentTheme.colors.surface.foreground }}>
|
||||||
|
{partialTranscript}
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<p className="typography-markdown md:typography-ui-label" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||||
|
{placeholderText}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
{status === 'failed' ? (
|
||||||
|
<p className="typography-meta mt-1" style={{ color: currentTheme.colors.status.error }}>
|
||||||
|
{error || t('chat.dictation.failed')}
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
|
{status === 'recording' && error && !isModelDownloading ? (
|
||||||
|
<p className="typography-meta mt-1" style={{ color: currentTheme.colors.status.warning }}>
|
||||||
|
{error}
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className={cn('flex flex-shrink-0 items-center gap-x-3', footerPaddingClass)}
|
||||||
|
style={footerHeight ? { height: footerHeight } : undefined}
|
||||||
|
>
|
||||||
|
{status === 'recording' ? (
|
||||||
|
<>
|
||||||
|
<span className="relative ml-1 flex h-2 w-2 flex-shrink-0" aria-hidden="true">
|
||||||
|
<span
|
||||||
|
className="absolute inline-flex h-full w-full animate-ping rounded-full opacity-60"
|
||||||
|
style={{ backgroundColor: currentTheme.colors.status.error }}
|
||||||
|
/>
|
||||||
|
<span
|
||||||
|
className="relative inline-flex h-2 w-2 rounded-full"
|
||||||
|
style={{ backgroundColor: currentTheme.colors.status.error }}
|
||||||
|
/>
|
||||||
|
</span>
|
||||||
|
<VolumeMeter volume={volume} />
|
||||||
|
<span className="typography-meta tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||||
|
{formatDuration(duration)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
) : status === 'uploading' ? (
|
||||||
|
<Icon name="loader-4" className="ml-1 h-4 w-4 animate-spin" style={{ color: currentTheme.colors.surface.mutedForeground }} />
|
||||||
|
) : null}
|
||||||
|
{/* Same inter-control gap as the composer's right cluster:
|
||||||
|
gap-x-1 on mobile, md:gap-x-3 on desktop. */}
|
||||||
|
<div className={cn('ml-auto flex items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
|
||||||
|
{status === 'recording' ? (
|
||||||
|
<>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
|
||||||
|
onClick={() => {
|
||||||
|
void cancelDictation();
|
||||||
|
}}
|
||||||
|
title={t('chat.dictation.cancel')}
|
||||||
|
aria-label={t('chat.dictation.cancel')}
|
||||||
|
>
|
||||||
|
<Icon name="close" className={iconSizeClass} />
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={footerIconButtonClass}
|
||||||
|
onClick={() => confirmWith('insert')}
|
||||||
|
title={t('chat.dictation.insert')}
|
||||||
|
aria-label={t('chat.dictation.insert')}
|
||||||
|
>
|
||||||
|
<Icon name="check" className={iconSizeClass} />
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={cn(footerIconButtonClass, 'text-primary hover:text-primary')}
|
||||||
|
onClick={() => confirmWith('send')}
|
||||||
|
title={t('chat.dictation.insertAndSend')}
|
||||||
|
aria-label={t('chat.dictation.insertAndSend')}
|
||||||
|
>
|
||||||
|
<Icon name="send-plane-2" className={sendIconSizeClass} />
|
||||||
|
</button>
|
||||||
|
</>
|
||||||
|
) : status === 'uploading' ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
|
||||||
|
onClick={() => {
|
||||||
|
void cancelDictation();
|
||||||
|
}}
|
||||||
|
title={t('chat.dictation.cancel')}
|
||||||
|
aria-label={t('chat.dictation.cancel')}
|
||||||
|
>
|
||||||
|
<Icon name="close" className={iconSizeClass} />
|
||||||
|
</button>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={cn(footerIconButtonClass, 'text-muted-foreground hover:text-foreground')}
|
||||||
|
onClick={discardFailedDictation}
|
||||||
|
title={t('chat.dictation.discard')}
|
||||||
|
aria-label={t('chat.dictation.discard')}
|
||||||
|
>
|
||||||
|
<Icon name="close" className={iconSizeClass} />
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={footerIconButtonClass}
|
||||||
|
onClick={retry}
|
||||||
|
title={t('chat.dictation.retry')}
|
||||||
|
aria-label={t('chat.dictation.retry')}
|
||||||
|
>
|
||||||
|
<Icon name="refresh" className={iconSizeClass} />
|
||||||
|
</button>
|
||||||
|
{partialTranscript.trim() ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={footerIconButtonClass}
|
||||||
|
onClick={() => {
|
||||||
|
pendingActionRef.current = 'insert';
|
||||||
|
acceptPartialTranscript();
|
||||||
|
}}
|
||||||
|
title={t('chat.dictation.insert')}
|
||||||
|
aria-label={t('chat.dictation.insert')}
|
||||||
|
>
|
||||||
|
<Icon name="check" className={iconSizeClass} />
|
||||||
|
</button>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -7,7 +7,6 @@ import type { ThemeMode } from '@/types/theme';
|
|||||||
import { useUIStore } from '@/stores/useUIStore';
|
import { useUIStore } from '@/stores/useUIStore';
|
||||||
import { useMessageQueueStore, type FollowUpBehavior } from '@/stores/messageQueueStore';
|
import { useMessageQueueStore, type FollowUpBehavior } from '@/stores/messageQueueStore';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
import { isCapacitorApp } from '@/lib/platform';
|
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { Checkbox } from '@/components/ui/checkbox';
|
import { Checkbox } from '@/components/ui/checkbox';
|
||||||
import { NumberInput } from '@/components/ui/number-input';
|
import { NumberInput } from '@/components/ui/number-input';
|
||||||
@@ -322,10 +321,7 @@ export const OpenChamberVisualSettings: React.FC<OpenChamberVisualSettingsProps>
|
|||||||
const setShowSplitAssistantMessageActions = useUIStore(state => state.setShowSplitAssistantMessageActions);
|
const setShowSplitAssistantMessageActions = useUIStore(state => state.setShowSplitAssistantMessageActions);
|
||||||
const messageStreamTransport = useConfigStore((state) => state.settingsMessageStreamTransport);
|
const messageStreamTransport = useConfigStore((state) => state.settingsMessageStreamTransport);
|
||||||
const setMessageStreamTransport = useConfigStore((state) => state.setSettingsMessageStreamTransport);
|
const setMessageStreamTransport = useConfigStore((state) => state.setSettingsMessageStreamTransport);
|
||||||
// Capacitor apps are locked to SSE (native WebSocket streaming is unreliable on mobile);
|
const effectiveMessageStreamTransport = messageStreamTransport;
|
||||||
// sync-context forces it too. Show SSE selected and disable the other options here.
|
|
||||||
const isCapacitorAppRuntime = React.useMemo(() => isCapacitorApp(), []);
|
|
||||||
const effectiveMessageStreamTransport = isCapacitorAppRuntime ? 'sse' : messageStreamTransport;
|
|
||||||
const settingsDefaultFileViewerPreview = useConfigStore((state) => state.settingsDefaultFileViewerPreview);
|
const settingsDefaultFileViewerPreview = useConfigStore((state) => state.settingsDefaultFileViewerPreview);
|
||||||
const setSettingsDefaultFileViewerPreview = useConfigStore((state) => state.setSettingsDefaultFileViewerPreview);
|
const setSettingsDefaultFileViewerPreview = useConfigStore((state) => state.setSettingsDefaultFileViewerPreview);
|
||||||
const isSettingsDialogOpen = useUIStore(state => state.isSettingsDialogOpen);
|
const isSettingsDialogOpen = useUIStore(state => state.isSettingsDialogOpen);
|
||||||
@@ -1531,7 +1527,6 @@ export const OpenChamberVisualSettings: React.FC<OpenChamberVisualSettingsProps>
|
|||||||
variant="chip"
|
variant="chip"
|
||||||
size="xs"
|
size="xs"
|
||||||
aria-pressed={effectiveMessageStreamTransport === option.id}
|
aria-pressed={effectiveMessageStreamTransport === option.id}
|
||||||
disabled={isCapacitorAppRuntime && option.id !== 'sse'}
|
|
||||||
className="!font-normal"
|
className="!font-normal"
|
||||||
onClick={() => handleMessageStreamTransportChange(option.id)}
|
onClick={() => handleMessageStreamTransportChange(option.id)}
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import React, { useEffect, useState, useCallback, useMemo } from 'react';
|
import React, { useEffect, useState, useCallback, useMemo } from 'react';
|
||||||
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
|
|
||||||
import { useConfigStore } from '@/stores/useConfigStore';
|
import { useConfigStore } from '@/stores/useConfigStore';
|
||||||
import { useDeviceInfo } from '@/lib/device';
|
import { useDeviceInfo } from '@/lib/device';
|
||||||
|
|
||||||
@@ -11,103 +10,362 @@ import {
|
|||||||
SelectValue,
|
SelectValue,
|
||||||
} from '@/components/ui/select';
|
} from '@/components/ui/select';
|
||||||
import { Checkbox } from '@/components/ui/checkbox';
|
import { Checkbox } from '@/components/ui/checkbox';
|
||||||
|
import { Radio } from '@/components/ui/radio';
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { NumberInput } from '@/components/ui/number-input';
|
import { NumberInput } from '@/components/ui/number-input';
|
||||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
||||||
import { Icon } from "@/components/icon/Icon";
|
import { Icon } from "@/components/icon/Icon";
|
||||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||||
import { audioStreamService } from '@/lib/voice/audioStreamService';
|
|
||||||
import { wasmSttService, WASM_MODELS } from '@/lib/voice/wasmSttService';
|
|
||||||
import type { WasmModelStatus } from '@/lib/voice/wasmSttService';
|
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||||
import { useI18n } from '@/lib/i18n';
|
import { useI18n } from '@/lib/i18n';
|
||||||
|
import { useLocalTTS } from '@/hooks/useLocalTTS';
|
||||||
import { disposePreviewAudio } from './voicePreviewAudio';
|
import { disposePreviewAudio } from './voicePreviewAudio';
|
||||||
const LANGUAGE_OPTIONS = [
|
|
||||||
{ value: 'en-US', label: 'English' },
|
|
||||||
{ value: 'es-ES', label: 'Español' },
|
|
||||||
{ value: 'fr-FR', label: 'Français' },
|
|
||||||
{ value: 'de-DE', label: 'Deutsch' },
|
|
||||||
{ value: 'ja-JP', label: '日本語' },
|
|
||||||
{ value: 'zh-CN', label: '中文' },
|
|
||||||
{ value: 'pt-BR', label: 'Português' },
|
|
||||||
{ value: 'it-IT', label: 'Italiano' },
|
|
||||||
{ value: 'ko-KR', label: '한국어' },
|
|
||||||
{ value: 'uk-UA', label: 'Українська' },
|
|
||||||
];
|
|
||||||
|
|
||||||
const WasmModelStatusIndicator = ({ modelId }: { modelId: string }) => {
|
const LOCAL_STT_MODELS = [
|
||||||
|
{
|
||||||
|
id: 'parakeet-tdt-0.6b-v2-int8',
|
||||||
|
labelKey: 'settings.voice.page.stt.model.parakeetV2',
|
||||||
|
badgeKey: 'settings.voice.page.stt.badge.bestForEnglish',
|
||||||
|
accuracy: 5,
|
||||||
|
speed: 5,
|
||||||
|
size: '460 MB',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'parakeet-tdt-0.6b-v3-int8',
|
||||||
|
labelKey: 'settings.voice.page.stt.model.parakeetV3',
|
||||||
|
badgeKey: 'settings.voice.page.stt.badge.bestForMultilingual',
|
||||||
|
accuracy: 5,
|
||||||
|
speed: 4,
|
||||||
|
size: '465 MB',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'whisper-base-int8',
|
||||||
|
labelKey: 'settings.voice.page.stt.model.whisperBase',
|
||||||
|
badgeKey: null,
|
||||||
|
accuracy: 3,
|
||||||
|
speed: 3,
|
||||||
|
size: '200 MB',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'whisper-tiny-int8',
|
||||||
|
labelKey: 'settings.voice.page.stt.model.whisperTiny',
|
||||||
|
badgeKey: null,
|
||||||
|
accuracy: 2,
|
||||||
|
speed: 4,
|
||||||
|
size: '115 MB',
|
||||||
|
},
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
interface DictationModelState {
|
||||||
|
id: string;
|
||||||
|
installed: boolean;
|
||||||
|
downloading: boolean;
|
||||||
|
downloadProgress: number | null;
|
||||||
|
downloadError: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RatingBar = ({ value, label }: { value: number; label: string }) => (
|
||||||
|
<span className="flex items-center gap-1.5" title={`${label}: ${value}/5`}>
|
||||||
|
<span
|
||||||
|
className="h-1.5 w-12 flex-shrink-0 overflow-hidden rounded-full bg-[var(--interactive-border)]"
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
className="block h-full rounded-full bg-[var(--primary-base)]"
|
||||||
|
style={{ width: `${(value / 5) * 100}%` }}
|
||||||
|
/>
|
||||||
|
</span>
|
||||||
|
<span className="typography-ui-compact text-muted-foreground">{label}</span>
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
|
||||||
|
const LocalModelPicker = ({
|
||||||
|
selectedModelId,
|
||||||
|
onSelect,
|
||||||
|
}: {
|
||||||
|
selectedModelId: string;
|
||||||
|
onSelect: (modelId: string) => void;
|
||||||
|
}) => {
|
||||||
const { t } = useI18n();
|
const { t } = useI18n();
|
||||||
const [status, setStatus] = useState<WasmModelStatus>(wasmSttService.getModelStatus());
|
const tUnsafe = useCallback((key: string) => t(key as Parameters<typeof t>[0]), [t]);
|
||||||
const [loading, setLoading] = useState(false);
|
const [models, setModels] = useState<Map<string, DictationModelState>>(new Map());
|
||||||
|
const [requestingId, setRequestingId] = useState<string | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
const refresh = useCallback(async () => {
|
||||||
const handler = (s: WasmModelStatus) => setStatus(s);
|
try {
|
||||||
wasmSttService.onModelStatusChange = handler;
|
const response = await runtimeFetch('/api/dictation/status', {
|
||||||
return () => { wasmSttService.onModelStatusChange = null; };
|
query: { provider: 'local' },
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const data = await response.json();
|
||||||
|
if (Array.isArray(data?.models)) {
|
||||||
|
setModels(new Map(data.models.map((m: DictationModelState) => [m.id, m])));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Display-only status; keep the previous state on fetch failure.
|
||||||
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const currentModelId = wasmSttService.getCurrentModelId();
|
|
||||||
const isLoadingOrDownloading = status.state === 'downloading' || status.state === 'loading';
|
|
||||||
|
|
||||||
// Reset local loading state when model finishes loading or errors
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (status.state !== 'downloading' && status.state !== 'loading') {
|
void refresh();
|
||||||
setLoading(false);
|
}, [refresh]);
|
||||||
}
|
|
||||||
}, [status.state]);
|
|
||||||
|
|
||||||
const handleDownload = async () => {
|
const anyDownloading = Array.from(models.values()).some((m) => m.downloading);
|
||||||
setLoading(true);
|
useEffect(() => {
|
||||||
|
if (!anyDownloading) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
void refresh();
|
||||||
|
}, 2000);
|
||||||
|
return () => clearInterval(interval);
|
||||||
|
}, [anyDownloading, refresh]);
|
||||||
|
|
||||||
|
const handleDownload = async (modelId: string) => {
|
||||||
|
setRequestingId(modelId);
|
||||||
try {
|
try {
|
||||||
await wasmSttService.loadModel(modelId);
|
await runtimeFetch(`/api/dictation/models/${encodeURIComponent(modelId)}/download`, {
|
||||||
|
method: 'POST',
|
||||||
|
});
|
||||||
|
await refresh();
|
||||||
} catch {
|
} catch {
|
||||||
// Error is shown via status indicator
|
// Status refresh reports errors.
|
||||||
|
} finally {
|
||||||
|
setRequestingId(null);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if (status.state === 'ready' && currentModelId === modelId) {
|
const handleDelete = async (modelId: string) => {
|
||||||
return (
|
setRequestingId(modelId);
|
||||||
<div className="flex items-center gap-2">
|
try {
|
||||||
<span className="typography-ui-compact text-green-600 dark:text-green-400">
|
await runtimeFetch(`/api/dictation/models/${encodeURIComponent(modelId)}`, {
|
||||||
{t('settings.voice.page.stt.wasmLoaded')}
|
method: 'DELETE',
|
||||||
</span>
|
});
|
||||||
</div>
|
await refresh();
|
||||||
);
|
} catch {
|
||||||
|
// Status refresh reports errors.
|
||||||
|
} finally {
|
||||||
|
setRequestingId(null);
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
if (isLoadingOrDownloading) {
|
|
||||||
const progress = status.state === 'downloading' ? Math.round(status.progress) : undefined;
|
|
||||||
return (
|
return (
|
||||||
<div className="flex items-center gap-2">
|
<div role="radiogroup" aria-label={t('settings.voice.page.field.model')} className="mt-1 space-y-0">
|
||||||
<span className="typography-ui-compact text-muted-foreground">
|
{LOCAL_STT_MODELS.map((entry) => {
|
||||||
{status.state === 'downloading' ? t('settings.voice.page.stt.wasmDownloading') : t('settings.voice.page.stt.wasmLoading')}
|
const selected = selectedModelId === entry.id;
|
||||||
{progress !== undefined ? ` (${progress}%)` : ''}
|
const state = models.get(entry.id) ?? null;
|
||||||
</span>
|
return (
|
||||||
|
<div key={entry.id} className="flex w-full items-start gap-2 py-1.5">
|
||||||
|
<div className="pt-0.5">
|
||||||
|
<Radio
|
||||||
|
checked={selected}
|
||||||
|
onChange={() => onSelect(entry.id)}
|
||||||
|
ariaLabel={tUnsafe(entry.labelKey)}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
);
|
<div
|
||||||
}
|
className="min-w-0 flex-1 cursor-pointer"
|
||||||
|
role="button"
|
||||||
if (status.state === 'error') {
|
tabIndex={0}
|
||||||
// Partial download (cached progress): show retry button.
|
onClick={() => onSelect(entry.id)}
|
||||||
return (
|
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); onSelect(entry.id); } }}
|
||||||
<div className="flex items-center gap-2">
|
>
|
||||||
<span className="typography-ui-compact text-destructive">{status.error}</span>
|
<div className="flex flex-wrap items-center gap-x-2 gap-y-0.5">
|
||||||
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
|
<span className={cn('typography-ui-label font-normal', selected ? 'text-foreground' : 'text-foreground/50')}>
|
||||||
{t('settings.voice.page.stt.wasmRetry')}
|
{tUnsafe(entry.labelKey)}
|
||||||
|
</span>
|
||||||
|
{entry.badgeKey ? (
|
||||||
|
<span
|
||||||
|
className="rounded px-1 text-[9px] font-medium uppercase leading-[14px] tracking-wide"
|
||||||
|
style={{
|
||||||
|
backgroundColor: 'var(--status-success-background)',
|
||||||
|
color: 'var(--status-success)',
|
||||||
|
border: '1px solid var(--status-success-border)',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{tUnsafe(entry.badgeKey)}
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
<div className="mt-0.5 flex flex-wrap items-center gap-x-4 gap-y-0.5">
|
||||||
|
<RatingBar value={entry.accuracy} label={t('settings.voice.page.stt.meta.accuracy')} />
|
||||||
|
<RatingBar value={entry.speed} label={t('settings.voice.page.stt.meta.speed')} />
|
||||||
|
<span className="typography-ui-compact tabular-nums text-muted-foreground">{entry.size}</span>
|
||||||
|
</div>
|
||||||
|
{state?.downloadError ? (
|
||||||
|
<p className="mt-0.5 typography-meta text-[var(--status-error)]">{state.downloadError}</p>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
<div className="flex w-14 flex-shrink-0 items-center justify-end gap-1 pt-0.5">
|
||||||
|
{state?.installed ? (
|
||||||
|
<>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="xs"
|
||||||
|
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
|
||||||
|
disabled={requestingId === entry.id}
|
||||||
|
onClick={() => { void handleDelete(entry.id); }}
|
||||||
|
title={t('settings.voice.page.stt.modelDelete')}
|
||||||
|
aria-label={t('settings.voice.page.stt.modelDelete')}
|
||||||
|
>
|
||||||
|
<Icon name="delete-bin" className="h-4 w-4" />
|
||||||
</Button>
|
</Button>
|
||||||
|
<Icon
|
||||||
|
name="checkbox-circle"
|
||||||
|
className="h-4 w-4 flex-shrink-0 text-[var(--status-success)]"
|
||||||
|
aria-label={t('settings.voice.page.stt.modelInstalled')}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
) : state?.downloading ? (
|
||||||
|
<span className="flex items-center gap-1.5">
|
||||||
|
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
|
||||||
|
<span className="typography-ui-compact tabular-nums text-muted-foreground">
|
||||||
|
{typeof state.downloadProgress === 'number' ? `${state.downloadProgress}%` : ''}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="xs"
|
||||||
|
className="h-6 w-6 p-0"
|
||||||
|
disabled={requestingId === entry.id}
|
||||||
|
onClick={() => { void handleDownload(entry.id); }}
|
||||||
|
title={t('settings.voice.page.stt.modelDownload')}
|
||||||
|
aria-label={t('settings.voice.page.stt.modelDownload')}
|
||||||
|
>
|
||||||
|
<Icon name="download" className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Kokoro en_v0_19 speaker ids in sherpa-onnx order. */
|
||||||
|
const KOKORO_VOICE_OPTIONS = [
|
||||||
|
{ id: 0, label: 'Alloy (af)' },
|
||||||
|
{ id: 1, label: 'Bella (af)' },
|
||||||
|
{ id: 2, label: 'Nicole (af)' },
|
||||||
|
{ id: 3, label: 'Sarah (af)' },
|
||||||
|
{ id: 4, label: 'Sky (af)' },
|
||||||
|
{ id: 5, label: 'Adam (am)' },
|
||||||
|
{ id: 6, label: 'Michael (am)' },
|
||||||
|
{ id: 7, label: 'Emma (bf)' },
|
||||||
|
{ id: 8, label: 'Isabella (bf)' },
|
||||||
|
{ id: 9, label: 'George (bm)' },
|
||||||
|
{ id: 10, label: 'Lewis (bm)' },
|
||||||
|
];
|
||||||
|
|
||||||
|
const LOCAL_TTS_MODEL_ID = 'kokoro-en-v0_19';
|
||||||
|
|
||||||
|
const LocalTtsModelStatus = () => {
|
||||||
|
const { t } = useI18n();
|
||||||
|
const [model, setModel] = useState<DictationModelState | null>(null);
|
||||||
|
const [requesting, setRequesting] = useState(false);
|
||||||
|
|
||||||
|
const refresh = useCallback(async () => {
|
||||||
|
try {
|
||||||
|
const response = await runtimeFetch('/api/dictation/status', { query: { provider: 'local' } });
|
||||||
|
if (!response.ok) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const data = await response.json();
|
||||||
|
const entry = Array.isArray(data?.ttsModels)
|
||||||
|
? data.ttsModels.find((m: DictationModelState) => m.id === LOCAL_TTS_MODEL_ID)
|
||||||
|
: null;
|
||||||
|
if (entry) {
|
||||||
|
setModel(entry);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Display-only status; keep the previous state on fetch failure.
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void refresh();
|
||||||
|
}, [refresh]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!model?.downloading) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
void refresh();
|
||||||
|
}, 2000);
|
||||||
|
return () => clearInterval(interval);
|
||||||
|
}, [model?.downloading, refresh]);
|
||||||
|
|
||||||
|
const request = async (method: 'POST' | 'DELETE') => {
|
||||||
|
setRequesting(true);
|
||||||
|
try {
|
||||||
|
const path = method === 'POST'
|
||||||
|
? `/api/dictation/models/${LOCAL_TTS_MODEL_ID}/download`
|
||||||
|
: `/api/dictation/models/${LOCAL_TTS_MODEL_ID}`;
|
||||||
|
await runtimeFetch(path, { method });
|
||||||
|
await refresh();
|
||||||
|
} catch {
|
||||||
|
// Status refresh reports errors.
|
||||||
|
} finally {
|
||||||
|
setRequesting(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!model) {
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center gap-2 py-1.5">
|
||||||
<span className="typography-ui-compact text-muted-foreground">
|
<span className="typography-ui-label text-foreground">Kokoro</span>
|
||||||
{t('settings.voice.page.stt.wasmNotLoaded')}
|
<span className="typography-ui-compact tabular-nums text-muted-foreground">305 MB</span>
|
||||||
</span>
|
{model.installed ? (
|
||||||
<Button variant="chip" size="xs" disabled={loading} onClick={handleDownload}>
|
<>
|
||||||
{t('settings.voice.page.stt.wasmDownload')}
|
<Icon
|
||||||
|
name="checkbox-circle"
|
||||||
|
className="h-4 w-4 text-[var(--status-success)]"
|
||||||
|
aria-label={t('settings.voice.page.stt.modelInstalled')}
|
||||||
|
/>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="xs"
|
||||||
|
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
|
||||||
|
disabled={requesting}
|
||||||
|
onClick={() => { void request('DELETE'); }}
|
||||||
|
title={t('settings.voice.page.stt.modelDelete')}
|
||||||
|
aria-label={t('settings.voice.page.stt.modelDelete')}
|
||||||
|
>
|
||||||
|
<Icon name="delete-bin" className="h-4 w-4" />
|
||||||
</Button>
|
</Button>
|
||||||
|
</>
|
||||||
|
) : model.downloading ? (
|
||||||
|
<span className="flex items-center gap-1.5">
|
||||||
|
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
|
||||||
|
<span className="typography-ui-compact tabular-nums text-muted-foreground">
|
||||||
|
{typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="xs"
|
||||||
|
className="h-6 w-6 p-0"
|
||||||
|
disabled={requesting}
|
||||||
|
onClick={() => { void request('POST'); }}
|
||||||
|
title={t('settings.voice.page.stt.modelDownload')}
|
||||||
|
aria-label={t('settings.voice.page.stt.modelDownload')}
|
||||||
|
>
|
||||||
|
<Icon name="download" className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
{model.downloadError ? (
|
||||||
|
<span className="typography-meta text-[var(--status-error)]">{model.downloadError}</span>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
@@ -131,11 +389,6 @@ const OPENAI_VOICE_OPTIONS = [
|
|||||||
export const VoiceSettings: React.FC = () => {
|
export const VoiceSettings: React.FC = () => {
|
||||||
const { t } = useI18n();
|
const { t } = useI18n();
|
||||||
const { isMobile } = useDeviceInfo();
|
const { isMobile } = useDeviceInfo();
|
||||||
const {
|
|
||||||
isSupported,
|
|
||||||
language,
|
|
||||||
setLanguage,
|
|
||||||
} = useBrowserVoice();
|
|
||||||
const voiceProvider = useConfigStore((state) => state.voiceProvider);
|
const voiceProvider = useConfigStore((state) => state.voiceProvider);
|
||||||
const setVoiceProvider = useConfigStore((state) => state.setVoiceProvider);
|
const setVoiceProvider = useConfigStore((state) => state.setVoiceProvider);
|
||||||
const speechRate = useConfigStore((state) => state.speechRate);
|
const speechRate = useConfigStore((state) => state.speechRate);
|
||||||
@@ -146,6 +399,22 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
const setSpeechVolume = useConfigStore((state) => state.setSpeechVolume);
|
const setSpeechVolume = useConfigStore((state) => state.setSpeechVolume);
|
||||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||||
const setSayVoice = useConfigStore((state) => state.setSayVoice);
|
const setSayVoice = useConfigStore((state) => state.setSayVoice);
|
||||||
|
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
|
||||||
|
const setLocalTtsVoiceId = useConfigStore((state) => state.setLocalTtsVoiceId);
|
||||||
|
const { speak: speakLocalTts, stop: stopLocalTts, isPlaying: isLocalTtsPlaying, error: localTtsError } = useLocalTTS();
|
||||||
|
|
||||||
|
const previewLocalVoice = useCallback(() => {
|
||||||
|
if (isLocalTtsPlaying) {
|
||||||
|
stopLocalTts();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const voiceLabel = KOKORO_VOICE_OPTIONS.find((v) => v.id === localTtsVoiceId)?.label
|
||||||
|
?? String(localTtsVoiceId);
|
||||||
|
void speakLocalTts(t('settings.voice.page.preview.voiceLine', { voiceName: voiceLabel }), {
|
||||||
|
speakerId: localTtsVoiceId,
|
||||||
|
speed: useConfigStore.getState().speechRate,
|
||||||
|
});
|
||||||
|
}, [isLocalTtsPlaying, localTtsVoiceId, speakLocalTts, stopLocalTts, t]);
|
||||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||||
const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice);
|
const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice);
|
||||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||||
@@ -172,19 +441,13 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
const setSttApiKey = useConfigStore((state) => state.setSttApiKey);
|
const setSttApiKey = useConfigStore((state) => state.setSttApiKey);
|
||||||
const sttModel = useConfigStore((state) => state.sttModel);
|
const sttModel = useConfigStore((state) => state.sttModel);
|
||||||
const setSttModel = useConfigStore((state) => state.setSttModel);
|
const setSttModel = useConfigStore((state) => state.setSttModel);
|
||||||
const wasmSttModel = useConfigStore((state) => state.wasmSttModel);
|
const sttLocalModel = useConfigStore((state) => state.sttLocalModel);
|
||||||
const setWasmSttModel = useConfigStore((state) => state.setWasmSttModel);
|
const setSttLocalModel = useConfigStore((state) => state.setSttLocalModel);
|
||||||
const sttLanguage = useConfigStore((state) => state.sttLanguage);
|
const sttLanguage = useConfigStore((state) => state.sttLanguage);
|
||||||
const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
|
const setSttLanguage = useConfigStore((state) => state.setSttLanguage);
|
||||||
const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb);
|
|
||||||
const setSttSilenceThresholdDb = useConfigStore((state) => state.setSttSilenceThresholdDb);
|
|
||||||
const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs);
|
|
||||||
const setSttSilenceHoldMs = useConfigStore((state) => state.setSttSilenceHoldMs);
|
|
||||||
const sttTranscribeOnStop = useConfigStore((state) => state.sttTranscribeOnStop);
|
|
||||||
const setSttTranscribeOnStop = useConfigStore((state) => state.setSttTranscribeOnStop);
|
|
||||||
const setShowMessageTTSButtons = useConfigStore((state) => state.setShowMessageTTSButtons);
|
const setShowMessageTTSButtons = useConfigStore((state) => state.setShowMessageTTSButtons);
|
||||||
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
|
const dictationEnabled = useConfigStore((state) => state.dictationEnabled);
|
||||||
const setVoiceModeEnabled = useConfigStore((state) => state.setVoiceModeEnabled);
|
const setDictationEnabled = useConfigStore((state) => state.setDictationEnabled);
|
||||||
|
|
||||||
const [isSayAvailable, setIsSayAvailable] = useState(false);
|
const [isSayAvailable, setIsSayAvailable] = useState(false);
|
||||||
const [sayVoices, setSayVoices] = useState<Array<{ name: string; locale: string }>>([]);
|
const [sayVoices, setSayVoices] = useState<Array<{ name: string; locale: string }>>([]);
|
||||||
@@ -274,7 +537,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
}, [isBrowserPreviewPlaying]);
|
}, [isBrowserPreviewPlaying]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!voiceModeEnabled || (voiceProvider !== 'openai' && voiceProvider !== 'openai-compatible')) {
|
if (!showMessageTTSButtons || (voiceProvider !== 'openai' && voiceProvider !== 'openai-compatible')) {
|
||||||
setIsOpenAIAvailable(openaiApiKey.trim().length > 0);
|
setIsOpenAIAvailable(openaiApiKey.trim().length > 0);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -292,10 +555,10 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
checkOpenAIAvailability();
|
checkOpenAIAvailability();
|
||||||
}, [openaiApiKey, voiceModeEnabled, voiceProvider]);
|
}, [openaiApiKey, showMessageTTSButtons, voiceProvider]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!voiceModeEnabled) {
|
if (!showMessageTTSButtons) {
|
||||||
setIsSayAvailable(false);
|
setIsSayAvailable(false);
|
||||||
setSayVoices([]);
|
setSayVoices([]);
|
||||||
return;
|
return;
|
||||||
@@ -317,7 +580,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
.catch(() => {
|
.catch(() => {
|
||||||
setIsSayAvailable(false);
|
setIsSayAvailable(false);
|
||||||
});
|
});
|
||||||
}, [voiceModeEnabled]);
|
}, [showMessageTTSButtons]);
|
||||||
|
|
||||||
const previewVoice = useCallback(async () => {
|
const previewVoice = useCallback(async () => {
|
||||||
if (previewAudio) {
|
if (previewAudio) {
|
||||||
@@ -498,11 +761,11 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
return (
|
return (
|
||||||
<div className="space-y-8">
|
<div className="space-y-8">
|
||||||
|
|
||||||
{/* Voice Setup */}
|
{/* Playback (read messages aloud) */}
|
||||||
<div data-settings-item="voice.voice-setup" className="mb-8">
|
<div data-settings-item="voice.playback" className="mb-8">
|
||||||
<div className="mb-1 px-1">
|
<div className="mb-1 px-1">
|
||||||
<h3 className="typography-ui-header font-medium text-foreground">
|
<h3 className="typography-ui-header font-medium text-foreground">
|
||||||
{t('settings.voice.page.section.voiceSetup')}
|
{t('settings.voice.page.section.playbackAndSummary')}
|
||||||
</h3>
|
</h3>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -512,15 +775,15 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
||||||
role="button"
|
role="button"
|
||||||
tabIndex={0}
|
tabIndex={0}
|
||||||
aria-pressed={voiceModeEnabled}
|
aria-pressed={showMessageTTSButtons}
|
||||||
onClick={() => setVoiceModeEnabled(!voiceModeEnabled)}
|
onClick={() => setShowMessageTTSButtons(!showMessageTTSButtons)}
|
||||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setVoiceModeEnabled(!voiceModeEnabled); } }}
|
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setShowMessageTTSButtons(!showMessageTTSButtons); } }}
|
||||||
>
|
>
|
||||||
<Checkbox checked={voiceModeEnabled} onChange={setVoiceModeEnabled} ariaLabel={t('settings.voice.page.field.enableVoiceModeAria')} />
|
<Checkbox checked={showMessageTTSButtons} onChange={setShowMessageTTSButtons} ariaLabel={t('settings.voice.page.field.messageReadAloudButtonAria')} />
|
||||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.enableVoiceMode')}</span>
|
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.messageReadAloudButton')}</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{voiceModeEnabled && (
|
{showMessageTTSButtons && (
|
||||||
<>
|
<>
|
||||||
<div className="pb-1.5 pt-0.5">
|
<div className="pb-1.5 pt-0.5">
|
||||||
<div className="flex min-w-0 flex-col gap-1.5">
|
<div className="flex min-w-0 flex-col gap-1.5">
|
||||||
@@ -533,6 +796,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
<TooltipContent sideOffset={8} className="max-w-xs">
|
<TooltipContent sideOffset={8} className="max-w-xs">
|
||||||
<ul className="space-y-1">
|
<ul className="space-y-1">
|
||||||
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.browser')}</li>
|
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.browser')}</li>
|
||||||
|
<li><strong>{t('settings.voice.page.provider.local')}</strong> {t('settings.voice.page.tooltip.localTts')}</li>
|
||||||
<li><strong>OpenAI:</strong> {t('settings.voice.page.tooltip.openai')}</li>
|
<li><strong>OpenAI:</strong> {t('settings.voice.page.tooltip.openai')}</li>
|
||||||
<li><strong>{t('settings.voice.page.provider.custom')}</strong> {t('settings.voice.page.tooltip.custom')}</li>
|
<li><strong>{t('settings.voice.page.provider.custom')}</strong> {t('settings.voice.page.tooltip.custom')}</li>
|
||||||
<li><strong>{t('settings.voice.page.provider.say')}</strong> {t('settings.voice.page.tooltip.say')}</li>
|
<li><strong>{t('settings.voice.page.provider.say')}</strong> {t('settings.voice.page.tooltip.say')}</li>
|
||||||
@@ -550,6 +814,15 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
>
|
>
|
||||||
{t('settings.voice.page.provider.browser')}
|
{t('settings.voice.page.provider.browser')}
|
||||||
</Button>
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="chip"
|
||||||
|
size="xs"
|
||||||
|
aria-pressed={voiceProvider === 'local'}
|
||||||
|
onClick={() => setVoiceProvider('local')}
|
||||||
|
className="!font-normal"
|
||||||
|
>
|
||||||
|
{t('settings.voice.page.provider.local')}
|
||||||
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
variant="chip"
|
variant="chip"
|
||||||
size="xs"
|
size="xs"
|
||||||
@@ -706,10 +979,39 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Local (Kokoro) TTS model status */}
|
||||||
|
{voiceProvider === 'local' && <LocalTtsModelStatus />}
|
||||||
|
|
||||||
{/* Voice Selection */}
|
{/* Voice Selection */}
|
||||||
<div className="flex items-center gap-8 py-1.5">
|
<div className="flex items-center gap-8 py-1.5">
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.voice')}</span>
|
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.voice')}</span>
|
||||||
<div className="flex items-center gap-2 w-fit">
|
<div className="flex items-center gap-2 w-fit">
|
||||||
|
{voiceProvider === 'local' && (
|
||||||
|
<>
|
||||||
|
<Select
|
||||||
|
value={String(localTtsVoiceId)}
|
||||||
|
onValueChange={(value) => setLocalTtsVoiceId(Number.parseInt(value, 10) || 0)}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="w-fit">
|
||||||
|
<SelectValue placeholder={t('settings.voice.page.field.selectVoicePlaceholder')}>
|
||||||
|
{(value) => KOKORO_VOICE_OPTIONS.find((v) => String(v.id) === value)?.label ?? value}
|
||||||
|
</SelectValue>
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{KOKORO_VOICE_OPTIONS.map((v) => (
|
||||||
|
<SelectItem key={v.id} value={String(v.id)}>{v.label}</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
<Button size="xs" variant="ghost" onClick={previewLocalVoice} title={t('settings.voice.page.actions.preview')}>
|
||||||
|
{isLocalTtsPlaying ? <Icon name="stop" className="w-3.5 h-3.5" /> : <Icon name="play" className="w-3.5 h-3.5" />}
|
||||||
|
</Button>
|
||||||
|
{localTtsError ? (
|
||||||
|
<span className="typography-meta text-[var(--status-error)]">{localTtsError}</span>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
{voiceProvider === 'openai' && isOpenAIAvailable && (
|
{voiceProvider === 'openai' && isOpenAIAvailable && (
|
||||||
<>
|
<>
|
||||||
<Select value={openaiVoice} onValueChange={setOpenaiVoice}>
|
<Select value={openaiVoice} onValueChange={setOpenaiVoice}>
|
||||||
@@ -775,7 +1077,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
<div className="flex items-center gap-8 py-1.5">
|
<div className="flex items-center gap-8 py-1.5">
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechRate')}</span>
|
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechRate')}</span>
|
||||||
<div className="flex items-center gap-2 w-fit">
|
<div className="flex items-center gap-2 w-fit">
|
||||||
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechRate} onChange={(e) => setSpeechRate(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
|
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechRate} onChange={(e) => setSpeechRate(Number(e.target.value))} className={sliderClass} />}
|
||||||
<NumberInput value={speechRate} onValueChange={setSpeechRate} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
|
<NumberInput value={speechRate} onValueChange={setSpeechRate} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -784,7 +1086,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
<div className="flex items-center gap-8 py-1.5">
|
<div className="flex items-center gap-8 py-1.5">
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechPitch')}</span>
|
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechPitch')}</span>
|
||||||
<div className="flex items-center gap-2 w-fit">
|
<div className="flex items-center gap-2 w-fit">
|
||||||
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechPitch} onChange={(e) => setSpeechPitch(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
|
{!isMobile && <input type="range" min={0.5} max={2} step={0.1} value={speechPitch} onChange={(e) => setSpeechPitch(Number(e.target.value))} className={sliderClass} />}
|
||||||
<NumberInput value={speechPitch} onValueChange={setSpeechPitch} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
|
<NumberInput value={speechPitch} onValueChange={setSpeechPitch} min={0.5} max={2} step={0.1} className="w-16 tabular-nums" />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -793,7 +1095,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
<div className="flex items-center gap-8 py-1.5">
|
<div className="flex items-center gap-8 py-1.5">
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechVolume')}</span>
|
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.speechVolume')}</span>
|
||||||
<div className="flex items-center gap-2 w-fit">
|
<div className="flex items-center gap-2 w-fit">
|
||||||
{!isMobile && <input type="range" min={0} max={1} step={0.1} value={speechVolume} onChange={(e) => setSpeechVolume(Number(e.target.value))} disabled={!isSupported} className={sliderClass} />}
|
{!isMobile && <input type="range" min={0} max={1} step={0.1} value={speechVolume} onChange={(e) => setSpeechVolume(Number(e.target.value))} className={sliderClass} />}
|
||||||
{isMobile ? (
|
{isMobile ? (
|
||||||
<NumberInput value={Math.round(speechVolume * 100)} onValueChange={(v) => setSpeechVolume(v / 100)} min={0} max={100} step={10} className="w-16 tabular-nums" />
|
<NumberInput value={Math.round(speechVolume * 100)} onValueChange={(v) => setSpeechVolume(v / 100)} min={0} max={100} step={10} className="w-16 tabular-nums" />
|
||||||
) : (
|
) : (
|
||||||
@@ -804,20 +1106,34 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Language */}
|
{/* TTS input mode */}
|
||||||
<div className="flex items-center gap-8 py-1.5">
|
<div className="pb-1.5 pt-0.5">
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.language')}</span>
|
<div className="flex min-w-0 flex-col gap-1.5">
|
||||||
<div className="flex items-center gap-2 w-fit">
|
<div className="flex items-center gap-1.5">
|
||||||
<Select value={language} onValueChange={setLanguage} disabled={!isSupported}>
|
<span className="typography-ui-label text-foreground">
|
||||||
<SelectTrigger className="w-fit">
|
{t('settings.voice.page.field.ttsInputMode')}
|
||||||
<SelectValue placeholder={t('settings.voice.page.field.selectLanguagePlaceholder')} />
|
</span>
|
||||||
</SelectTrigger>
|
</div>
|
||||||
<SelectContent>
|
<div className="flex flex-wrap items-center gap-1">
|
||||||
{LANGUAGE_OPTIONS.map((lang) => (
|
<Button
|
||||||
<SelectItem key={lang.value} value={lang.value}>{lang.label}</SelectItem>
|
variant="chip"
|
||||||
))}
|
size="xs"
|
||||||
</SelectContent>
|
aria-pressed={ttsInputMode === 'sanitized'}
|
||||||
</Select>
|
onClick={() => setTtsInputMode('sanitized')}
|
||||||
|
className="!font-normal"
|
||||||
|
>
|
||||||
|
{t('settings.voice.page.field.ttsInputModeSanitized')}
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="chip"
|
||||||
|
size="xs"
|
||||||
|
aria-pressed={ttsInputMode === 'raw'}
|
||||||
|
onClick={() => setTtsInputMode('raw')}
|
||||||
|
className="!font-normal"
|
||||||
|
>
|
||||||
|
{t('settings.voice.page.field.ttsInputModeRaw')}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
@@ -826,7 +1142,6 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Speech Recognition */}
|
{/* Speech Recognition */}
|
||||||
{voiceModeEnabled && (
|
|
||||||
<div data-settings-item="voice.speech-recognition" className="mb-8">
|
<div data-settings-item="voice.speech-recognition" className="mb-8">
|
||||||
<div className="mb-1 px-1">
|
<div className="mb-1 px-1">
|
||||||
<h3 className="typography-ui-header font-medium text-foreground">
|
<h3 className="typography-ui-header font-medium text-foreground">
|
||||||
@@ -835,6 +1150,19 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<section className="px-2 pb-2 pt-0 space-y-0">
|
<section className="px-2 pb-2 pt-0 space-y-0">
|
||||||
|
<div
|
||||||
|
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
||||||
|
role="button"
|
||||||
|
tabIndex={0}
|
||||||
|
aria-pressed={dictationEnabled}
|
||||||
|
onClick={() => setDictationEnabled(!dictationEnabled)}
|
||||||
|
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setDictationEnabled(!dictationEnabled); } }}
|
||||||
|
>
|
||||||
|
<Checkbox checked={dictationEnabled} onChange={setDictationEnabled} ariaLabel={t('settings.voice.page.field.enableVoiceInputAria')} />
|
||||||
|
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.enableVoiceInput')}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{dictationEnabled && (<>
|
||||||
<div className="pb-1.5 pt-0.5">
|
<div className="pb-1.5 pt-0.5">
|
||||||
<div className="flex min-w-0 flex-col gap-1.5">
|
<div className="flex min-w-0 flex-col gap-1.5">
|
||||||
<div className="flex items-center gap-1.5">
|
<div className="flex items-center gap-1.5">
|
||||||
@@ -845,7 +1173,7 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
</TooltipTrigger>
|
</TooltipTrigger>
|
||||||
<TooltipContent sideOffset={8} className="max-w-xs">
|
<TooltipContent sideOffset={8} className="max-w-xs">
|
||||||
<ul className="space-y-1">
|
<ul className="space-y-1">
|
||||||
<li><strong>{t('settings.voice.page.provider.browser')}</strong> {t('settings.voice.page.tooltip.sttBrowser')}</li>
|
<li><strong>{t('settings.voice.page.provider.local')}</strong> {t('settings.voice.page.tooltip.sttLocal')}</li>
|
||||||
<li><strong>{t('settings.voice.page.provider.server')}</strong> {t('settings.voice.page.tooltip.sttServer')}</li>
|
<li><strong>{t('settings.voice.page.provider.server')}</strong> {t('settings.voice.page.tooltip.sttServer')}</li>
|
||||||
</ul>
|
</ul>
|
||||||
</TooltipContent>
|
</TooltipContent>
|
||||||
@@ -855,53 +1183,34 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
<Button
|
<Button
|
||||||
variant="chip"
|
variant="chip"
|
||||||
size="xs"
|
size="xs"
|
||||||
aria-pressed={sttProvider === 'browser'}
|
aria-pressed={sttProvider === 'local'}
|
||||||
onClick={() => setSttProvider('browser')}
|
onClick={() => setSttProvider('local')}
|
||||||
className="!font-normal"
|
className="!font-normal"
|
||||||
>
|
>
|
||||||
{t('settings.voice.page.provider.browser')}
|
{t('settings.voice.page.provider.local')}
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
variant="chip"
|
variant="chip"
|
||||||
size="xs"
|
size="xs"
|
||||||
aria-pressed={sttProvider === 'server'}
|
aria-pressed={sttProvider === 'openai-compatible'}
|
||||||
onClick={() => setSttProvider('server')}
|
onClick={() => setSttProvider('openai-compatible')}
|
||||||
className="!font-normal"
|
className="!font-normal"
|
||||||
>
|
>
|
||||||
{t('settings.voice.page.provider.server')}
|
{t('settings.voice.page.provider.server')}
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
|
||||||
variant="chip"
|
|
||||||
size="xs"
|
|
||||||
aria-pressed={sttProvider === 'wasm'}
|
|
||||||
onClick={() => setSttProvider('wasm')}
|
|
||||||
className="!font-normal"
|
|
||||||
>
|
|
||||||
{t('settings.voice.page.provider.wasm')}
|
|
||||||
</Button>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{sttProvider === 'server' && (
|
{sttProvider === 'local' && (
|
||||||
<div className="py-1.5 space-y-2">
|
<div className="py-1.5">
|
||||||
<div
|
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.model')}</span>
|
||||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
<LocalModelPicker selectedModelId={sttLocalModel} onSelect={setSttLocalModel} />
|
||||||
role="button"
|
|
||||||
tabIndex={0}
|
|
||||||
aria-pressed={sttTranscribeOnStop}
|
|
||||||
onClick={() => setSttTranscribeOnStop(!sttTranscribeOnStop)}
|
|
||||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setSttTranscribeOnStop(!sttTranscribeOnStop); } }}
|
|
||||||
>
|
|
||||||
<Checkbox checked={sttTranscribeOnStop} onChange={setSttTranscribeOnStop} ariaLabel={t('settings.voice.page.field.transcribeOnStopAria')} />
|
|
||||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.transcribeOnStop')}</span>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{!audioStreamService.isSupported() && (
|
|
||||||
<p className="typography-meta text-[var(--status-error)]">
|
|
||||||
{t('settings.voice.page.field.sttBrowserSupportError')}
|
|
||||||
</p>
|
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{sttProvider === 'openai-compatible' && (
|
||||||
|
<div className="py-1.5 space-y-2">
|
||||||
<div>
|
<div>
|
||||||
<span className={cn("typography-ui-label text-foreground", !sttServerUrl.trim() && "text-[var(--status-error)]")}>
|
<span className={cn("typography-ui-label text-foreground", !sttServerUrl.trim() && "text-[var(--status-error)]")}>
|
||||||
{t('settings.voice.page.field.serverUrl')}
|
{t('settings.voice.page.field.serverUrl')}
|
||||||
@@ -979,128 +1288,13 @@ export const VoiceSettings: React.FC = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="flex items-center gap-8 py-0.5">
|
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.silenceThreshold')}</span>
|
|
||||||
<div className="flex items-center gap-2 w-fit">
|
|
||||||
{!isMobile && <input type="range" min={-60} max={-20} step={1} value={sttSilenceThresholdDb} onChange={(e) => setSttSilenceThresholdDb(Number(e.target.value))} className={sliderClass} />}
|
|
||||||
<span className="typography-ui-label text-foreground tabular-nums min-w-[3.5rem] text-right">
|
|
||||||
{sttSilenceThresholdDb} dB
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div className="flex items-center gap-8 py-0.5">
|
|
||||||
<span className="typography-ui-label text-foreground sm:w-56 shrink-0">{t('settings.voice.page.field.silenceHold')}</span>
|
|
||||||
<div className="flex items-center gap-2 w-fit">
|
|
||||||
{!isMobile && <input type="range" min={500} max={3000} step={100} value={sttSilenceHoldMs} onChange={(e) => setSttSilenceHoldMs(Number(e.target.value))} className={sliderClass} />}
|
|
||||||
<NumberInput value={sttSilenceHoldMs} onValueChange={setSttSilenceHoldMs} min={500} max={3000} step={100} className="w-20 tabular-nums" />
|
|
||||||
<span className="typography-meta text-muted-foreground">{t('settings.voice.page.field.millisecondsUnit')}</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{sttProvider === 'wasm' && (
|
</>)}
|
||||||
<div className="py-1.5 space-y-2">
|
|
||||||
<div>
|
|
||||||
<span className="typography-ui-label text-muted-foreground">
|
|
||||||
{t('settings.voice.page.stt.wasmModel')}
|
|
||||||
</span>
|
|
||||||
<Select value={wasmSttModel} onValueChange={setWasmSttModel}>
|
|
||||||
<SelectTrigger className="mt-0.5">
|
|
||||||
<SelectValue placeholder={t('settings.voice.page.stt.wasmModel')} />
|
|
||||||
</SelectTrigger>
|
|
||||||
<SelectContent>
|
|
||||||
{WASM_MODELS.map((m) => (
|
|
||||||
<SelectItem key={m.id} value={m.id}>
|
|
||||||
<div className="flex flex-col">
|
|
||||||
<span>{m.name}</span>
|
|
||||||
<span className="typography-ui-compact text-muted-foreground">
|
|
||||||
{m.size} · {m.languages}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</SelectItem>
|
|
||||||
))}
|
|
||||||
</SelectContent>
|
|
||||||
</Select>
|
|
||||||
<p className="typography-ui-compact text-muted-foreground mt-0.5">
|
|
||||||
{WASM_MODELS.find((m) => m.id === wasmSttModel)?.description}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
{/* Model status indicator */}
|
|
||||||
<WasmModelStatusIndicator modelId={wasmSttModel} />
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</section>
|
</section>
|
||||||
</div>
|
</div>
|
||||||
)}
|
|
||||||
|
|
||||||
{/* Playback & Summarization */}
|
|
||||||
<div data-settings-item="voice.playback" className="mb-8">
|
|
||||||
<div className="mb-1 px-1">
|
|
||||||
<h3 className="typography-ui-header font-medium text-foreground">
|
|
||||||
{t('settings.voice.page.section.playbackAndSummary')}
|
|
||||||
</h3>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<section className="px-2 pb-2 pt-0 space-y-0">
|
|
||||||
<div
|
|
||||||
className="group flex cursor-pointer items-center gap-2 py-1.5"
|
|
||||||
role="button"
|
|
||||||
tabIndex={0}
|
|
||||||
aria-pressed={showMessageTTSButtons}
|
|
||||||
onClick={() => setShowMessageTTSButtons(!showMessageTTSButtons)}
|
|
||||||
onKeyDown={(e) => { if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); setShowMessageTTSButtons(!showMessageTTSButtons); } }}
|
|
||||||
>
|
|
||||||
<Checkbox checked={showMessageTTSButtons} onChange={setShowMessageTTSButtons} ariaLabel={t('settings.voice.page.field.messageReadAloudButtonAria')} />
|
|
||||||
<span className="typography-ui-label text-foreground">{t('settings.voice.page.field.messageReadAloudButton')}</span>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="pb-1.5 pt-0.5">
|
|
||||||
<div className="flex min-w-0 flex-col gap-1.5">
|
|
||||||
<div className="flex items-center gap-1.5">
|
|
||||||
<span className="typography-ui-label text-foreground">
|
|
||||||
{t('settings.voice.page.field.ttsInputMode')}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<div className="flex flex-wrap items-center gap-1">
|
|
||||||
<Button
|
|
||||||
variant="chip"
|
|
||||||
size="xs"
|
|
||||||
aria-pressed={ttsInputMode === 'sanitized'}
|
|
||||||
onClick={() => setTtsInputMode('sanitized')}
|
|
||||||
className="!font-normal"
|
|
||||||
>
|
|
||||||
{t('settings.voice.page.field.ttsInputModeSanitized')}
|
|
||||||
</Button>
|
|
||||||
<Button
|
|
||||||
variant="chip"
|
|
||||||
size="xs"
|
|
||||||
aria-pressed={ttsInputMode === 'raw'}
|
|
||||||
onClick={() => setTtsInputMode('raw')}
|
|
||||||
className="!font-normal"
|
|
||||||
>
|
|
||||||
{t('settings.voice.page.field.ttsInputModeRaw')}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
{voiceModeEnabled && isSupported && (
|
|
||||||
<div className="mt-2 px-2">
|
|
||||||
<p className="typography-meta text-muted-foreground">
|
|
||||||
{t('settings.voice.page.hint.shiftClickPrefix')}
|
|
||||||
{' '}
|
|
||||||
<kbd className="px-1 py-0.5 mx-0.5 rounded border border-[var(--interactive-border)] bg-background typography-mono text-[10px]">Shift</kbd>
|
|
||||||
{' + '}
|
|
||||||
<kbd className="px-1 py-0.5 mx-0.5 rounded border border-[var(--interactive-border)] bg-background typography-mono text-[10px]">Click</kbd>
|
|
||||||
{' '}
|
|
||||||
{t('settings.voice.page.hint.shiftClickSuffix')}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1030,7 +1030,7 @@ export const SettingsView: React.FC<SettingsViewProps> = ({ onClose, forceMobile
|
|||||||
: <Icon name={iconName!} className="h-4 w-4 shrink-0" />}
|
: <Icon name={iconName!} className="h-4 w-4 shrink-0" />}
|
||||||
<span className="flex items-center gap-1.5 whitespace-nowrap overflow-hidden transition-opacity duration-150 opacity-100">
|
<span className="flex items-center gap-1.5 whitespace-nowrap overflow-hidden transition-opacity duration-150 opacity-100">
|
||||||
<span className="typography-ui-label font-normal truncate">{getPageTitle(page.slug)}</span>
|
<span className="typography-ui-label font-normal truncate">{getPageTitle(page.slug)}</span>
|
||||||
{(page.slug === 'voice' || page.slug === 'tunnel') && (
|
{page.slug === 'tunnel' && (
|
||||||
<span className="shrink-0 typography-micro px-1 rounded leading-none pb-px text-[var(--status-warning)] bg-[var(--status-warning)]/10">
|
<span className="shrink-0 typography-micro px-1 rounded leading-none pb-px text-[var(--status-warning)] bg-[var(--status-warning)]/10">
|
||||||
{t('settings.view.badge.beta')}
|
{t('settings.view.badge.beta')}
|
||||||
</span>
|
</span>
|
||||||
|
|||||||
@@ -1,392 +0,0 @@
|
|||||||
/**
|
|
||||||
* BrowserVoiceButton Component
|
|
||||||
*
|
|
||||||
* Voice toggle button for browser-based voice chat with language selection.
|
|
||||||
* Shows visual state indicators for different voice modes.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```tsx
|
|
||||||
* <BrowserVoiceButton />
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
|
||||||
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
|
|
||||||
import { useConfigStore } from '@/stores/useConfigStore';
|
|
||||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
|
||||||
import { isVSCodeRuntime } from '@/lib/desktop';
|
|
||||||
import { Button } from '@/components/ui/button';
|
|
||||||
import {
|
|
||||||
Tooltip,
|
|
||||||
TooltipContent,
|
|
||||||
TooltipProvider,
|
|
||||||
TooltipTrigger,
|
|
||||||
} from '@/components/ui/tooltip';
|
|
||||||
import { VoiceStatusIndicator } from './VoiceStatusIndicator';
|
|
||||||
import { toast } from '@/components/ui/toast';
|
|
||||||
import { Icon } from "@/components/icon/Icon";
|
|
||||||
import { useI18n } from '@/lib/i18n';
|
|
||||||
|
|
||||||
// Status text for accessibility and labels
|
|
||||||
const statusLabels: Record<string, string> = {
|
|
||||||
idle: 'Start Voice',
|
|
||||||
listening: 'Listening',
|
|
||||||
processing: 'Processing',
|
|
||||||
speaking: 'AI Speaking',
|
|
||||||
error: 'Voice Error',
|
|
||||||
};
|
|
||||||
|
|
||||||
// iOS Safari detection utility
|
|
||||||
const isIOSSafari = (): boolean => {
|
|
||||||
if (typeof navigator === 'undefined') return false;
|
|
||||||
const userAgent = navigator.userAgent.toLowerCase();
|
|
||||||
const isIOS = /iphone|ipad|ipod/i.test(userAgent);
|
|
||||||
const isSafari = /safari/i.test(userAgent) && !/chrome|crios|crmo/i.test(userAgent);
|
|
||||||
return isIOS && isSafari;
|
|
||||||
};
|
|
||||||
|
|
||||||
const normalizeVoiceErrorMessage = (error: string): string => {
|
|
||||||
const isMediaDevicesError =
|
|
||||||
error.includes('getUserMedia') ||
|
|
||||||
error.includes('mediaDevices') ||
|
|
||||||
error.includes('Cannot read properties of undefined');
|
|
||||||
|
|
||||||
if (!isMediaDevicesError) {
|
|
||||||
return error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof window !== 'undefined' && !window.isSecureContext) {
|
|
||||||
return 'Voice requires a secure connection (HTTPS) or localhost. Please use HTTPS or access via localhost.';
|
|
||||||
}
|
|
||||||
|
|
||||||
return 'Microphone access is unavailable in this runtime. On desktop, check System Settings -> Privacy & Security -> Microphone for OpenChamber.';
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Browser Voice Button with language selection
|
|
||||||
*/
|
|
||||||
export function BrowserVoiceButton() {
|
|
||||||
const { t } = useI18n();
|
|
||||||
const voiceModeEnabled = useConfigStore((s) => s.voiceModeEnabled);
|
|
||||||
const sttProvider = useConfigStore((s) => s.sttProvider);
|
|
||||||
const sttTranscribeOnStop = useConfigStore((s) => s.sttTranscribeOnStop);
|
|
||||||
|
|
||||||
const {
|
|
||||||
status,
|
|
||||||
isSupported,
|
|
||||||
error,
|
|
||||||
|
|
||||||
startVoice,
|
|
||||||
stopVoice,
|
|
||||||
finishVoiceInput,
|
|
||||||
conversationMode,
|
|
||||||
toggleConversationMode,
|
|
||||||
isMobile,
|
|
||||||
} = useBrowserVoice();
|
|
||||||
|
|
||||||
const [isPressing, setIsPressing] = useState(false);
|
|
||||||
const isVSCode = isVSCodeRuntime();
|
|
||||||
const buttonSizeClass = isMobile ? 'h-8 w-8 min-h-[32px] min-w-[32px]' : (isVSCode ? 'h-5 w-5' : 'h-6 w-6');
|
|
||||||
const iconSizeClass = isMobile ? 'h-[18px] w-[18px]' : (isVSCode ? 'h-4 w-4' : 'h-[18px] w-[18px]');
|
|
||||||
const continuousIconSizeClass = 'size-[18px]';
|
|
||||||
const clearHoverBackgroundClass = 'bg-transparent hover:!bg-transparent active:!bg-transparent focus-visible:!bg-transparent';
|
|
||||||
|
|
||||||
// Refs for touch handling
|
|
||||||
const touchHandledRef = useRef(false);
|
|
||||||
const isIOSSafariRef = useRef(false);
|
|
||||||
const longPressTimerRef = useRef<NodeJS.Timeout | null>(null);
|
|
||||||
const longPressTriggeredRef = useRef(false);
|
|
||||||
const lastToastedErrorRef = useRef<string | null>(null);
|
|
||||||
|
|
||||||
// Initialize iOS detection on mount
|
|
||||||
useEffect(() => {
|
|
||||||
isIOSSafariRef.current = isIOSSafari();
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// NOTE: Do NOT pre-request microphone permission on mount.
|
|
||||||
// Permission is requested when the user explicitly taps the mic button.
|
|
||||||
// Pre-requesting causes an unwanted permission prompt on mobile page load.
|
|
||||||
|
|
||||||
// Determine active states
|
|
||||||
const isActive = status === 'listening' || status === 'speaking' || status === 'processing';
|
|
||||||
const isError = status === 'error';
|
|
||||||
const isIdle = status === 'idle';
|
|
||||||
|
|
||||||
const isSpeaking = status === 'speaking';
|
|
||||||
// WASM STT always needs finishVoiceInput to flush the recorder and transcribe.
|
|
||||||
// Server STT uses it when sttTranscribeOnStop is enabled.
|
|
||||||
const canTranscribeOnStop = sttProvider === 'wasm' || (sttProvider === 'server' && sttTranscribeOnStop);
|
|
||||||
const isListeningWithTranscribeOnStop = status === 'listening' && canTranscribeOnStop;
|
|
||||||
|
|
||||||
// Show toast notification when voice error occurs
|
|
||||||
useEffect(() => {
|
|
||||||
if (isError && error) {
|
|
||||||
if (lastToastedErrorRef.current === error) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
lastToastedErrorRef.current = error;
|
|
||||||
const displayError = normalizeVoiceErrorMessage(error);
|
|
||||||
|
|
||||||
toast.error(displayError, {
|
|
||||||
duration: 5000,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isError) {
|
|
||||||
lastToastedErrorRef.current = null;
|
|
||||||
}
|
|
||||||
}, [isError, error]);
|
|
||||||
|
|
||||||
// Status text for accessibility
|
|
||||||
const statusText = isError
|
|
||||||
? error || 'Voice Error'
|
|
||||||
: isListeningWithTranscribeOnStop
|
|
||||||
? t('voice.action.finishAndTranscribe')
|
|
||||||
: conversationMode && status === 'idle'
|
|
||||||
? 'Start Voice (Continuous mode on)'
|
|
||||||
: statusLabels[status] || 'Start Voice';
|
|
||||||
|
|
||||||
// Tooltip content based on state
|
|
||||||
const getTooltipContent = () => {
|
|
||||||
if (isError && error) {
|
|
||||||
return normalizeVoiceErrorMessage(error);
|
|
||||||
}
|
|
||||||
if (isListeningWithTranscribeOnStop) {
|
|
||||||
return t('voice.action.finishAndTranscribe');
|
|
||||||
}
|
|
||||||
if (isActive) {
|
|
||||||
return 'Stop voice conversation';
|
|
||||||
}
|
|
||||||
if (isMobile) {
|
|
||||||
return 'Start voice conversation';
|
|
||||||
}
|
|
||||||
return `Start voice conversation (Shift+Click for continuous mode) • Cmd/Ctrl+Shift+V to toggle`;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Handle voice activation (used by both click and touch)
|
|
||||||
const activateVoice = useCallback(async () => {
|
|
||||||
if (isActive) {
|
|
||||||
if (status === 'listening' && canTranscribeOnStop) {
|
|
||||||
finishVoiceInput();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
stopVoice();
|
|
||||||
} else if (status !== 'error') {
|
|
||||||
// On mobile, we must NOT do any async operations before calling startVoice()
|
|
||||||
// because iOS Safari requires SpeechRecognition.start() to be called
|
|
||||||
// synchronously within the user gesture handler
|
|
||||||
if (isMobile) {
|
|
||||||
// Start voice immediately - no await before this!
|
|
||||||
// Audio unlock is now handled inside startVoice() for mobile
|
|
||||||
startVoice();
|
|
||||||
} else {
|
|
||||||
// Desktop can use async path
|
|
||||||
try {
|
|
||||||
await startVoice();
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to start voice:', err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Reset from error state
|
|
||||||
if (isMobile) {
|
|
||||||
startVoice();
|
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
await startVoice();
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to start voice:', err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, [isActive, status, canTranscribeOnStop, finishVoiceInput, startVoice, stopVoice, isMobile]);
|
|
||||||
|
|
||||||
// Handle Shift+Click to toggle conversation mode
|
|
||||||
const handleClick = useCallback(async (e: React.MouseEvent) => {
|
|
||||||
// Prevent double-firing if touch already handled this
|
|
||||||
if (touchHandledRef.current) {
|
|
||||||
touchHandledRef.current = false;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Shift+Click toggles conversation mode
|
|
||||||
if (e.shiftKey) {
|
|
||||||
toggleConversationMode();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await activateVoice();
|
|
||||||
}, [activateVoice, toggleConversationMode]);
|
|
||||||
|
|
||||||
// Handle touch start for mobile devices
|
|
||||||
const handleTouchStart = useCallback((e: React.TouchEvent) => {
|
|
||||||
// Prevent default to stop mouse event emulation
|
|
||||||
e.preventDefault();
|
|
||||||
|
|
||||||
// Mark that touch handled this interaction
|
|
||||||
touchHandledRef.current = true;
|
|
||||||
longPressTriggeredRef.current = false;
|
|
||||||
|
|
||||||
// Immediate visual feedback
|
|
||||||
setIsPressing(true);
|
|
||||||
|
|
||||||
// Set up long-press timer for toggling conversation mode (500ms)
|
|
||||||
longPressTimerRef.current = setTimeout(() => {
|
|
||||||
longPressTriggeredRef.current = true;
|
|
||||||
toggleConversationMode();
|
|
||||||
// Haptic feedback if available
|
|
||||||
if (navigator.vibrate) {
|
|
||||||
navigator.vibrate(50);
|
|
||||||
}
|
|
||||||
setIsPressing(false);
|
|
||||||
}, 500);
|
|
||||||
}, [toggleConversationMode]);
|
|
||||||
|
|
||||||
// Handle touch end
|
|
||||||
const handleTouchEnd = useCallback(() => {
|
|
||||||
// Clear long-press timer
|
|
||||||
if (longPressTimerRef.current) {
|
|
||||||
clearTimeout(longPressTimerRef.current);
|
|
||||||
longPressTimerRef.current = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only activate voice if long-press wasn't triggered
|
|
||||||
if (!longPressTriggeredRef.current) {
|
|
||||||
activateVoice();
|
|
||||||
}
|
|
||||||
|
|
||||||
setIsPressing(false);
|
|
||||||
}, [activateVoice]);
|
|
||||||
|
|
||||||
// Handle touch cancel
|
|
||||||
const handleTouchCancel = useCallback(() => {
|
|
||||||
if (longPressTimerRef.current) {
|
|
||||||
clearTimeout(longPressTimerRef.current);
|
|
||||||
longPressTimerRef.current = null;
|
|
||||||
}
|
|
||||||
setIsPressing(false);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
const handleToggleConversationMode = useCallback((event: React.MouseEvent<HTMLButtonElement>) => {
|
|
||||||
event.preventDefault();
|
|
||||||
event.stopPropagation();
|
|
||||||
toggleConversationMode();
|
|
||||||
}, [toggleConversationMode]);
|
|
||||||
|
|
||||||
// If voice mode is disabled, don't render anything
|
|
||||||
if (!voiceModeEnabled) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If not supported, show disabled button with tooltip
|
|
||||||
if (!isSupported) {
|
|
||||||
const supportDetails = browserVoiceService.getSupportDetails();
|
|
||||||
const tooltipMessage = !supportDetails.secureContext
|
|
||||||
? 'Voice requires HTTPS or localhost. Please use a secure connection.'
|
|
||||||
: !supportDetails.recognition
|
|
||||||
? 'Speech recognition not supported in this browser. Try Chrome, Edge, or Safari.'
|
|
||||||
: !supportDetails.synthesis
|
|
||||||
? 'Speech synthesis not supported in this browser.'
|
|
||||||
: 'Voice not supported in this browser';
|
|
||||||
|
|
||||||
return (
|
|
||||||
<TooltipProvider>
|
|
||||||
<Tooltip>
|
|
||||||
<TooltipTrigger asChild>
|
|
||||||
<Button
|
|
||||||
size="icon"
|
|
||||||
variant="ghost"
|
|
||||||
disabled
|
|
||||||
aria-label={tooltipMessage}
|
|
||||||
className={`${buttonSizeClass} p-0 ${clearHoverBackgroundClass}`}
|
|
||||||
>
|
|
||||||
<Icon name="mic-off" className={`${iconSizeClass} opacity-50`} />
|
|
||||||
</Button>
|
|
||||||
</TooltipTrigger>
|
|
||||||
<TooltipContent side="bottom" align="center">
|
|
||||||
<p className="max-w-[200px] text-center">{tooltipMessage}</p>
|
|
||||||
</TooltipContent>
|
|
||||||
</Tooltip>
|
|
||||||
</TooltipProvider>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className={`flex items-center ${isMobile ? 'gap-1' : 'gap-1.5'}`}>
|
|
||||||
{/* Status indicator with label - show when active, simplified on mobile */}
|
|
||||||
{isActive && !isMobile && (
|
|
||||||
<VoiceStatusIndicator
|
|
||||||
status={status}
|
|
||||||
showLabel
|
|
||||||
size="sm"
|
|
||||||
className="mr-1"
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{/* Voice button with tooltip */}
|
|
||||||
<TooltipProvider>
|
|
||||||
<Tooltip>
|
|
||||||
<TooltipTrigger asChild>
|
|
||||||
<Button
|
|
||||||
size="icon"
|
|
||||||
variant="ghost"
|
|
||||||
onClick={handleClick}
|
|
||||||
onTouchStart={handleTouchStart}
|
|
||||||
onTouchEnd={handleTouchEnd}
|
|
||||||
onTouchCancel={handleTouchCancel}
|
|
||||||
aria-label={statusText}
|
|
||||||
className={`
|
|
||||||
relative
|
|
||||||
${buttonSizeClass}
|
|
||||||
p-0
|
|
||||||
${clearHoverBackgroundClass}
|
|
||||||
touch-manipulation
|
|
||||||
${isPressing ? 'scale-95 opacity-80' : ''}
|
|
||||||
${conversationMode && isIdle && isMobile ? 'ring-1 ring-primary/50' : ''}
|
|
||||||
`}
|
|
||||||
style={{
|
|
||||||
WebkitTapHighlightColor: 'transparent',
|
|
||||||
touchAction: 'manipulation',
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{isActive ? (
|
|
||||||
isSpeaking ? (
|
|
||||||
// Green speaker icon when AI is speaking
|
|
||||||
<Icon name="volume-up" className={`${iconSizeClass} text-green-400 animate-pulse`} />
|
|
||||||
) : (
|
|
||||||
// Red stop icon for listening/processing (both mobile and desktop)
|
|
||||||
<Icon name="stop-circle" className={`${iconSizeClass} text-[var(--status-error)]`} />
|
|
||||||
)
|
|
||||||
) : (
|
|
||||||
<VoiceStatusIndicator
|
|
||||||
status={isError ? 'idle' : status}
|
|
||||||
size={isMobile || isVSCode ? 'sm' : 'md'}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</Button>
|
|
||||||
</TooltipTrigger>
|
|
||||||
<TooltipContent side="bottom" align="center">
|
|
||||||
<p className="max-w-[200px] text-center">{getTooltipContent()}</p>
|
|
||||||
</TooltipContent>
|
|
||||||
</Tooltip>
|
|
||||||
</TooltipProvider>
|
|
||||||
|
|
||||||
{/* Conversation mode toggle button */}
|
|
||||||
{(status === 'idle' || status === 'error') && (
|
|
||||||
<Button
|
|
||||||
size="icon"
|
|
||||||
variant="ghost"
|
|
||||||
onPointerDownCapture={(event) => event.stopPropagation()}
|
|
||||||
onClick={handleToggleConversationMode}
|
|
||||||
aria-label={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
|
|
||||||
title={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
|
|
||||||
className={
|
|
||||||
`${buttonSizeClass} p-0 ${clearHoverBackgroundClass} ${conversationMode ? 'text-[var(--status-info)] hover:text-[var(--status-info)]' : 'text-muted-foreground hover:text-foreground'}`
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<Icon name="voice-recognition" className={continuousIconSizeClass} />
|
|
||||||
</Button>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
import React from 'react';
|
|
||||||
import { useVoiceContext } from '@/hooks/useVoiceContext';
|
|
||||||
import { useConfigStore } from '@/stores/useConfigStore';
|
|
||||||
|
|
||||||
const VoiceContextBridge = React.memo(function VoiceContextBridge() {
|
|
||||||
useVoiceContext();
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Provider component that initializes voice context sync.
|
|
||||||
* Wrap the app with this to enable voice session awareness.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```tsx
|
|
||||||
* <VoiceProvider>
|
|
||||||
* <App />
|
|
||||||
* </VoiceProvider>
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
export function VoiceProvider({ children }: { children: React.ReactNode }) {
|
|
||||||
const voiceModeEnabled = useConfigStore((state) => state.voiceModeEnabled);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<>
|
|
||||||
{voiceModeEnabled ? <VoiceContextBridge /> : null}
|
|
||||||
{children}
|
|
||||||
</>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,139 +0,0 @@
|
|||||||
/**
|
|
||||||
* VoiceStatusIndicator Component
|
|
||||||
*
|
|
||||||
* Reusable visual indicator for voice mode states with icons, animations,
|
|
||||||
* and optional status text labels.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```tsx
|
|
||||||
* // Basic usage - icon only
|
|
||||||
* <VoiceStatusIndicator status="listening" />
|
|
||||||
*
|
|
||||||
* // With label
|
|
||||||
* <VoiceStatusIndicator status="listening" showLabel />
|
|
||||||
*
|
|
||||||
* // Different size
|
|
||||||
* <VoiceStatusIndicator status="processing" size="lg" />
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
import React from 'react';
|
|
||||||
import type { BrowserVoiceStatus } from '@/hooks/useBrowserVoice';
|
|
||||||
import { useI18n } from '@/lib/i18n';
|
|
||||||
import { Icon } from "@/components/icon/Icon";
|
|
||||||
import type { IconName } from "@/components/icon/icons";
|
|
||||||
|
|
||||||
export interface VoiceStatusIndicatorProps {
|
|
||||||
/** Current voice status */
|
|
||||||
status: BrowserVoiceStatus;
|
|
||||||
/** Show text label next to icon */
|
|
||||||
showLabel?: boolean;
|
|
||||||
/** Size of the indicator */
|
|
||||||
size?: 'sm' | 'md' | 'lg';
|
|
||||||
/** Optional className for styling */
|
|
||||||
className?: string;
|
|
||||||
/** Whether conversation mode is active (shows indicator dot when idle) */
|
|
||||||
conversationMode?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sizeClasses = {
|
|
||||||
sm: {
|
|
||||||
icon: 'w-4 h-4',
|
|
||||||
container: 'gap-1.5',
|
|
||||||
},
|
|
||||||
md: {
|
|
||||||
icon: 'w-5 h-5',
|
|
||||||
container: 'gap-2',
|
|
||||||
},
|
|
||||||
lg: {
|
|
||||||
icon: 'w-6 h-6',
|
|
||||||
container: 'gap-2.5',
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const statusConfig: Record<
|
|
||||||
BrowserVoiceStatus,
|
|
||||||
{
|
|
||||||
icon: IconName;
|
|
||||||
color: string;
|
|
||||||
labelKey:
|
|
||||||
| 'voice.status.idle'
|
|
||||||
| 'voice.status.listening'
|
|
||||||
| 'voice.status.processing'
|
|
||||||
| 'voice.status.speaking'
|
|
||||||
| 'voice.status.error';
|
|
||||||
animation?: string;
|
|
||||||
}
|
|
||||||
> = {
|
|
||||||
idle: {
|
|
||||||
icon: "mic-off",
|
|
||||||
color: 'text-muted-foreground',
|
|
||||||
labelKey: 'voice.status.idle',
|
|
||||||
},
|
|
||||||
listening: {
|
|
||||||
icon: "mic",
|
|
||||||
color: 'text-primary',
|
|
||||||
labelKey: 'voice.status.listening',
|
|
||||||
animation: 'animate-pulse',
|
|
||||||
},
|
|
||||||
processing: {
|
|
||||||
icon: "loader-4",
|
|
||||||
color: 'text-primary',
|
|
||||||
labelKey: 'voice.status.processing',
|
|
||||||
animation: 'animate-spin',
|
|
||||||
},
|
|
||||||
speaking: {
|
|
||||||
icon: "volume-up",
|
|
||||||
color: 'text-green-500',
|
|
||||||
labelKey: 'voice.status.speaking',
|
|
||||||
},
|
|
||||||
error: {
|
|
||||||
icon: "alert",
|
|
||||||
color: 'text-destructive',
|
|
||||||
labelKey: 'voice.status.error',
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* VoiceStatusIndicator - Visual indicator for voice mode states
|
|
||||||
*/
|
|
||||||
export function VoiceStatusIndicator({
|
|
||||||
status,
|
|
||||||
showLabel = false,
|
|
||||||
size = 'md',
|
|
||||||
className = '',
|
|
||||||
conversationMode = false,
|
|
||||||
}: VoiceStatusIndicatorProps) {
|
|
||||||
const { t } = useI18n();
|
|
||||||
const config = statusConfig[status];
|
|
||||||
const statusIconName = config.icon;
|
|
||||||
const sizeClass = sizeClasses[size];
|
|
||||||
const containerClass = showLabel ? sizeClass.container : '';
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className={`flex items-center ${containerClass} ${className}`}>
|
|
||||||
<div className="relative">
|
|
||||||
<Icon name={statusIconName}
|
|
||||||
className={`
|
|
||||||
${sizeClass.icon}
|
|
||||||
${config.color}
|
|
||||||
${config.animation || ''}
|
|
||||||
`}
|
|
||||||
aria-hidden="true"
|
|
||||||
/>
|
|
||||||
{/* Conversation mode indicator dot - only when idle and conversation mode is on */}
|
|
||||||
{conversationMode && status === 'idle' && (
|
|
||||||
<span
|
|
||||||
className="absolute -top-0.5 -right-0.5 w-2 h-2 bg-green-500 rounded-full"
|
|
||||||
aria-label={t('voice.status.conversationModeActiveAria')}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
{showLabel && (
|
|
||||||
<span className={`typography-meta ${config.color}`}>
|
|
||||||
{t(config.labelKey)}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
export { VoiceProvider } from './VoiceProvider';
|
|
||||||
export { BrowserVoiceButton } from './BrowserVoiceButton';
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,408 @@
|
|||||||
|
/**
|
||||||
|
* Streaming dictation state machine.
|
||||||
|
*
|
||||||
|
* Status flow: idle -> recording -> uploading -> idle | failed.
|
||||||
|
* While recording, mic PCM chunks stream to the server, which sends back live
|
||||||
|
* partial transcripts. Confirm finalizes and resolves the full text; failed
|
||||||
|
* dictations retain their audio segments so retry can replay them.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
|
|
||||||
|
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
|
||||||
|
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
|
||||||
|
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
|
||||||
|
import { useConfigStore } from '@/stores/useConfigStore';
|
||||||
|
|
||||||
|
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
|
||||||
|
|
||||||
|
export interface UseDictationOptions {
|
||||||
|
onTranscript?: (text: string) => void;
|
||||||
|
onError?: (error: Error) => void;
|
||||||
|
canStart?: () => boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UseDictationResult {
|
||||||
|
status: DictationStatus;
|
||||||
|
isRecording: boolean;
|
||||||
|
isProcessing: boolean;
|
||||||
|
partialTranscript: string;
|
||||||
|
volume: number;
|
||||||
|
duration: number;
|
||||||
|
error: string | null;
|
||||||
|
errorReason: string | null;
|
||||||
|
startDictation: () => Promise<void>;
|
||||||
|
confirmDictation: () => Promise<string | null>;
|
||||||
|
cancelDictation: () => Promise<void>;
|
||||||
|
retryFailedDictation: () => Promise<string | null>;
|
||||||
|
acceptPartialTranscript: () => string | null;
|
||||||
|
discardFailedDictation: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DURATION_TICK_MS = 1000;
|
||||||
|
|
||||||
|
const toError = (value: unknown): Error =>
|
||||||
|
value instanceof Error ? value : new Error(String(value));
|
||||||
|
|
||||||
|
const getDictationStartOptions = (): DictationStartOptions => {
|
||||||
|
const state = useConfigStore.getState();
|
||||||
|
const language = state.sttLanguage?.trim();
|
||||||
|
if (state.sttProvider === 'openai-compatible') {
|
||||||
|
return {
|
||||||
|
provider: 'openai-compatible',
|
||||||
|
...(language ? { language } : {}),
|
||||||
|
openaiCompatible: {
|
||||||
|
baseUrl: state.sttServerUrl,
|
||||||
|
model: state.sttModel,
|
||||||
|
...(state.sttApiKey ? { apiKey: state.sttApiKey } : {}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
provider: 'local',
|
||||||
|
...(language ? { language } : {}),
|
||||||
|
localModel: state.sttLocalModel,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export function useDictation(options: UseDictationOptions = {}): UseDictationResult {
|
||||||
|
const { onTranscript, onError, canStart } = options;
|
||||||
|
|
||||||
|
const [status, setStatus] = useState<DictationStatus>('idle');
|
||||||
|
const [partialTranscript, setPartialTranscript] = useState('');
|
||||||
|
const [duration, setDuration] = useState(0);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [errorReason, setErrorReason] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const statusRef = useRef(status);
|
||||||
|
useEffect(() => {
|
||||||
|
statusRef.current = status;
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
|
const latestPartialRef = useRef('');
|
||||||
|
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
const actionGateRef = useRef({ starting: false, confirming: false, cancelling: false });
|
||||||
|
|
||||||
|
const onTranscriptRef = useRef(onTranscript);
|
||||||
|
const onErrorRef = useRef(onError);
|
||||||
|
useEffect(() => {
|
||||||
|
onTranscriptRef.current = onTranscript;
|
||||||
|
onErrorRef.current = onError;
|
||||||
|
}, [onTranscript, onError]);
|
||||||
|
|
||||||
|
const senderRef = useRef<DictationStreamSender | null>(null);
|
||||||
|
if (!senderRef.current) {
|
||||||
|
senderRef.current = new DictationStreamSender({
|
||||||
|
client: dictationClient,
|
||||||
|
getStartOptions: getDictationStartOptions,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const stopDurationTracking = useCallback(() => {
|
||||||
|
if (durationIntervalRef.current) {
|
||||||
|
clearInterval(durationIntervalRef.current);
|
||||||
|
durationIntervalRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const startDurationTracking = useCallback(() => {
|
||||||
|
if (durationIntervalRef.current) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
durationIntervalRef.current = setInterval(() => {
|
||||||
|
setDuration((prev) => prev + 1);
|
||||||
|
}, DURATION_TICK_MS);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const reportError = useCallback((err: unknown) => {
|
||||||
|
const normalized = toError(err);
|
||||||
|
setError(normalized.message);
|
||||||
|
const reason = (normalized as Error & { reasonCode?: string }).reasonCode;
|
||||||
|
setErrorReason(typeof reason === 'string' ? reason : null);
|
||||||
|
onErrorRef.current?.(normalized);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const clearError = useCallback(() => {
|
||||||
|
setError(null);
|
||||||
|
setErrorReason(null);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const clearStreamingState = useCallback(() => {
|
||||||
|
senderRef.current?.clearAll();
|
||||||
|
latestPartialRef.current = '';
|
||||||
|
setPartialTranscript('');
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Live partial transcripts for the active dictation.
|
||||||
|
useEffect(() => {
|
||||||
|
return dictationClient.onPartial((dictationId, text) => {
|
||||||
|
const activeDictationId = senderRef.current?.getDictationId();
|
||||||
|
if (!activeDictationId || dictationId !== activeDictationId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
latestPartialRef.current = text;
|
||||||
|
setPartialTranscript(text);
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Restart the stream (replaying buffered segments) after a reconnect.
|
||||||
|
useEffect(() => {
|
||||||
|
return dictationClient.subscribeConnectionStatus((connected) => {
|
||||||
|
if (!connected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (statusRef.current !== 'recording') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
void senderRef.current?.restartStream().catch((err) => {
|
||||||
|
reportError(err);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}, [reportError]);
|
||||||
|
|
||||||
|
const audio = useDictationAudioSource({
|
||||||
|
onPcmSegment: (audioData) => {
|
||||||
|
senderRef.current?.enqueueSegment(audioData);
|
||||||
|
},
|
||||||
|
onError: (err) => {
|
||||||
|
onErrorRef.current?.(err);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const audioStopRef = useRef(audio.stop);
|
||||||
|
useEffect(() => {
|
||||||
|
audioStopRef.current = audio.stop;
|
||||||
|
}, [audio.stop]);
|
||||||
|
|
||||||
|
const handleSuccess = useCallback(
|
||||||
|
(text: string): string | null => {
|
||||||
|
setDuration(0);
|
||||||
|
setStatus('idle');
|
||||||
|
const transcriptText = text.trim().length > 0 ? text.trim() : latestPartialRef.current.trim();
|
||||||
|
clearStreamingState();
|
||||||
|
if (!transcriptText) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
onTranscriptRef.current?.(transcriptText);
|
||||||
|
return transcriptText;
|
||||||
|
},
|
||||||
|
[clearStreamingState],
|
||||||
|
);
|
||||||
|
|
||||||
|
const handleFailure = useCallback(
|
||||||
|
(failure: unknown) => {
|
||||||
|
if (senderRef.current?.hasSegments()) {
|
||||||
|
setStatus('failed');
|
||||||
|
} else {
|
||||||
|
setStatus('idle');
|
||||||
|
}
|
||||||
|
reportError(failure);
|
||||||
|
},
|
||||||
|
[reportError],
|
||||||
|
);
|
||||||
|
|
||||||
|
const startDictation = useCallback(async () => {
|
||||||
|
const gate = actionGateRef.current;
|
||||||
|
if (gate.starting || gate.confirming || gate.cancelling) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (statusRef.current !== 'idle') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (canStart && !canStart()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
gate.starting = true;
|
||||||
|
clearError();
|
||||||
|
setPartialTranscript('');
|
||||||
|
setDuration(0);
|
||||||
|
setStatus('recording');
|
||||||
|
statusRef.current = 'recording';
|
||||||
|
clearStreamingState();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await audio.start();
|
||||||
|
startDurationTracking();
|
||||||
|
// Open the stream eagerly so partials start flowing immediately.
|
||||||
|
await senderRef.current?.restartStream().catch((err) => {
|
||||||
|
// Non-fatal: segments buffer locally and finish() retries the
|
||||||
|
// start, but surface the reason (e.g. model downloading) so
|
||||||
|
// the overlay can show it.
|
||||||
|
reportError(err);
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
await audio.stop().catch(() => undefined);
|
||||||
|
stopDurationTracking();
|
||||||
|
setStatus('idle');
|
||||||
|
statusRef.current = 'idle';
|
||||||
|
reportError(err);
|
||||||
|
} finally {
|
||||||
|
gate.starting = false;
|
||||||
|
}
|
||||||
|
}, [audio, canStart, clearError, clearStreamingState, reportError, startDurationTracking, stopDurationTracking]);
|
||||||
|
|
||||||
|
const cancelDictation = useCallback(async () => {
|
||||||
|
const gate = actionGateRef.current;
|
||||||
|
if (gate.cancelling) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (statusRef.current !== 'recording' && statusRef.current !== 'uploading') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
gate.cancelling = true;
|
||||||
|
stopDurationTracking();
|
||||||
|
setDuration(0);
|
||||||
|
clearError();
|
||||||
|
|
||||||
|
// Optimistic: dismiss the overlay immediately. Tearing down the audio
|
||||||
|
// graph (AudioContext.close) can take up to ~1s on mobile WebViews and
|
||||||
|
// must not delay the visible response to Cancel.
|
||||||
|
setStatus('idle');
|
||||||
|
statusRef.current = 'idle';
|
||||||
|
try {
|
||||||
|
senderRef.current?.cancel();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
clearStreamingState();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await audio.stop();
|
||||||
|
} catch {
|
||||||
|
// Cancelled anyway; mic teardown failures are not user-actionable.
|
||||||
|
} finally {
|
||||||
|
gate.cancelling = false;
|
||||||
|
}
|
||||||
|
}, [audio, clearError, clearStreamingState, stopDurationTracking]);
|
||||||
|
|
||||||
|
const confirmDictation = useCallback(async (): Promise<string | null> => {
|
||||||
|
const gate = actionGateRef.current;
|
||||||
|
if (gate.confirming) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (statusRef.current !== 'recording') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
gate.confirming = true;
|
||||||
|
clearError();
|
||||||
|
stopDurationTracking();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await audio.stop();
|
||||||
|
setStatus('uploading');
|
||||||
|
statusRef.current = 'uploading';
|
||||||
|
|
||||||
|
const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
|
||||||
|
if (finalSeq < 0) {
|
||||||
|
return handleSuccess('');
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await senderRef.current!.finish(finalSeq);
|
||||||
|
return handleSuccess(result.text);
|
||||||
|
} catch (err) {
|
||||||
|
handleFailure(err);
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
gate.confirming = false;
|
||||||
|
}
|
||||||
|
}, [audio, clearError, handleFailure, handleSuccess, stopDurationTracking]);
|
||||||
|
|
||||||
|
const retryFailedDictation = useCallback(async (): Promise<string | null> => {
|
||||||
|
if (statusRef.current !== 'failed' || !senderRef.current?.hasSegments()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
clearError();
|
||||||
|
setStatus('uploading');
|
||||||
|
statusRef.current = 'uploading';
|
||||||
|
|
||||||
|
try {
|
||||||
|
senderRef.current.resetStreamForReplay();
|
||||||
|
const finalSeq = senderRef.current.getFinalSeq();
|
||||||
|
const result = await senderRef.current.finish(finalSeq);
|
||||||
|
return handleSuccess(result.text);
|
||||||
|
} catch (err) {
|
||||||
|
handleFailure(err);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}, [clearError, handleFailure, handleSuccess]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Failed dictations still hold the last streamed partial transcript.
|
||||||
|
* Accept it as-is instead of retrying the full transcription.
|
||||||
|
*/
|
||||||
|
const acceptPartialTranscript = useCallback((): string | null => {
|
||||||
|
if (statusRef.current !== 'failed') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const text = latestPartialRef.current.trim();
|
||||||
|
setDuration(0);
|
||||||
|
setStatus('idle');
|
||||||
|
statusRef.current = 'idle';
|
||||||
|
clearError();
|
||||||
|
try {
|
||||||
|
senderRef.current?.cancel();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
clearStreamingState();
|
||||||
|
if (!text) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
onTranscriptRef.current?.(text);
|
||||||
|
return text;
|
||||||
|
}, [clearError, clearStreamingState]);
|
||||||
|
|
||||||
|
const discardFailedDictation = useCallback(() => {
|
||||||
|
setDuration(0);
|
||||||
|
setStatus('idle');
|
||||||
|
statusRef.current = 'idle';
|
||||||
|
clearError();
|
||||||
|
clearStreamingState();
|
||||||
|
}, [clearError, clearStreamingState]);
|
||||||
|
|
||||||
|
// While recording without an open stream (e.g. the model is still
|
||||||
|
// downloading), retry the stream start so live partials kick in as soon
|
||||||
|
// as the provider becomes ready. Buffered segments replay on success.
|
||||||
|
useEffect(() => {
|
||||||
|
if (status !== 'recording' || errorReason !== 'model_download_in_progress') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
if (statusRef.current !== 'recording' || senderRef.current?.getDictationId()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
senderRef.current?.restartStream().then(() => {
|
||||||
|
clearError();
|
||||||
|
}).catch((err) => {
|
||||||
|
reportError(err);
|
||||||
|
});
|
||||||
|
}, 3000);
|
||||||
|
return () => clearInterval(interval);
|
||||||
|
}, [status, errorReason, clearError, reportError]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
stopDurationTracking();
|
||||||
|
void audioStopRef.current().catch(() => undefined);
|
||||||
|
senderRef.current?.cancel();
|
||||||
|
};
|
||||||
|
}, [stopDurationTracking]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
isRecording: status === 'recording',
|
||||||
|
isProcessing: status === 'uploading',
|
||||||
|
partialTranscript,
|
||||||
|
volume: audio.volume,
|
||||||
|
duration,
|
||||||
|
error,
|
||||||
|
errorReason,
|
||||||
|
startDictation,
|
||||||
|
confirmDictation,
|
||||||
|
cancelDictation,
|
||||||
|
retryFailedDictation,
|
||||||
|
acceptPartialTranscript,
|
||||||
|
discardFailedDictation,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -474,6 +474,18 @@ export const useKeyboardShortcuts = () => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (eventMatchesShortcut(e, combo('toggle_dictation'))) {
|
||||||
|
const { activeMainTab, isCommandPaletteOpen, isHelpDialogOpen, isSessionSwitcherOpen, isSettingsDialogOpen } = useUIStore.getState();
|
||||||
|
if (activeMainTab !== 'chat' || isCommandPaletteOpen || isHelpDialogOpen || isSessionSwitcherOpen || isSettingsDialogOpen) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
e.preventDefault();
|
||||||
|
// Dictation state lives inside the composer's isolated component;
|
||||||
|
// toggle it via an event instead of subscribing this hot hook to it.
|
||||||
|
window.dispatchEvent(new CustomEvent('openchamber:dictation-toggle'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (e.key === 'Escape') {
|
if (e.key === 'Escape') {
|
||||||
const target = e.target as Element | null;
|
const target = e.target as Element | null;
|
||||||
const isInsideDialog = Boolean(target?.closest('[role="dialog"]'));
|
const isInsideDialog = Boolean(target?.closest('[role="dialog"]'));
|
||||||
|
|||||||
@@ -0,0 +1,266 @@
|
|||||||
|
/**
|
||||||
|
* useLocalTTS Hook
|
||||||
|
*
|
||||||
|
* React hook for local (Kokoro via sherpa-onnx) text-to-speech playback.
|
||||||
|
* Synthesis runs on the OpenChamber server in the dictation worker.
|
||||||
|
*
|
||||||
|
* Long texts are pipelined by sentence chunks: the first chunk starts playing
|
||||||
|
* as soon as it is synthesized while the next chunk synthesizes in the
|
||||||
|
* background, so time-to-first-audio stays ~1 chunk regardless of message
|
||||||
|
* length.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
|
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||||
|
|
||||||
|
export interface LocalTTSSpeakOptions {
|
||||||
|
/** Kokoro speaker id (0-10) */
|
||||||
|
speakerId?: number;
|
||||||
|
/** Playback speed multiplier (1.0 = normal) */
|
||||||
|
speed?: number;
|
||||||
|
onStart?: () => void;
|
||||||
|
onEnd?: () => void;
|
||||||
|
onError?: (error: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UseLocalTTSReturn {
|
||||||
|
isPlaying: boolean;
|
||||||
|
error: string | null;
|
||||||
|
speak: (text: string, options?: LocalTTSSpeakOptions) => Promise<void>;
|
||||||
|
stop: () => void;
|
||||||
|
/** Unlock audio for mobile Safari - call this on user gesture */
|
||||||
|
unlockAudio: () => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Target chunk size: big enough to amortize requests, small enough for low latency. */
|
||||||
|
const MIN_CHUNK_CHARS = 60;
|
||||||
|
const MAX_CHUNK_CHARS = 400;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split text into sentence-aligned chunks for pipelined synthesis.
|
||||||
|
* Sentences are merged until MIN_CHUNK_CHARS and hard-split at
|
||||||
|
* MAX_CHUNK_CHARS so a single run-on sentence cannot stall the pipeline.
|
||||||
|
*/
|
||||||
|
export function splitTextForSynthesis(text: string): string[] {
|
||||||
|
const normalized = text.replace(/\s+/g, ' ').trim();
|
||||||
|
if (!normalized) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const sentences = normalized.match(/[^.!?…]+[.!?…]+["')\]]*\s*|[^.!?…]+$/g) ?? [normalized];
|
||||||
|
|
||||||
|
const chunks: string[] = [];
|
||||||
|
let current = '';
|
||||||
|
for (const sentence of sentences) {
|
||||||
|
for (let offset = 0; offset < sentence.length; offset += MAX_CHUNK_CHARS) {
|
||||||
|
const piece = sentence.slice(offset, offset + MAX_CHUNK_CHARS);
|
||||||
|
if (current && current.length + piece.length > MAX_CHUNK_CHARS) {
|
||||||
|
chunks.push(current.trim());
|
||||||
|
current = '';
|
||||||
|
}
|
||||||
|
current += piece;
|
||||||
|
if (current.length >= MIN_CHUNK_CHARS && /[.!?…]["')\]]*\s*$/.test(current)) {
|
||||||
|
chunks.push(current.trim());
|
||||||
|
current = '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (current.trim()) {
|
||||||
|
chunks.push(current.trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Latency trim for the FIRST chunk only: if the opening sentence is long,
|
||||||
|
// split it at a clause boundary (comma/semicolon/colon) so the first
|
||||||
|
// audio starts sooner. Only the first chunk gets this treatment — comma
|
||||||
|
// splits inside every sentence would chop the prosody, and after playback
|
||||||
|
// starts the pipeline hides synthesis time anyway.
|
||||||
|
if (chunks.length > 0 && chunks[0].length > 120) {
|
||||||
|
const first = chunks[0];
|
||||||
|
const clauseBreak = /[,;:]\s/g;
|
||||||
|
let bestBreak = -1;
|
||||||
|
let match: RegExpExecArray | null;
|
||||||
|
while ((match = clauseBreak.exec(first)) !== null) {
|
||||||
|
const end = match.index + 1;
|
||||||
|
if (end < 40) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (end > 160) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bestBreak = end;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (bestBreak > 0) {
|
||||||
|
const head = first.slice(0, bestBreak).trim();
|
||||||
|
const tail = first.slice(bestBreak).trim();
|
||||||
|
chunks.splice(0, 1, head, tail);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunks;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sharedAudioContext: AudioContext | null = null;
|
||||||
|
|
||||||
|
function getAudioContext(): AudioContext {
|
||||||
|
if (!sharedAudioContext) {
|
||||||
|
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||||
|
}
|
||||||
|
return sharedAudioContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PlaybackSession {
|
||||||
|
cancelled: boolean;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useLocalTTS(): UseLocalTTSReturn {
|
||||||
|
const [isPlaying, setIsPlaying] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||||
|
const sessionRef = useRef<PlaybackSession | null>(null);
|
||||||
|
|
||||||
|
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||||
|
try {
|
||||||
|
const ctx = getAudioContext();
|
||||||
|
if (ctx.state === 'suspended') {
|
||||||
|
await ctx.resume();
|
||||||
|
}
|
||||||
|
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||||
|
const source = ctx.createBufferSource();
|
||||||
|
source.buffer = buffer;
|
||||||
|
source.connect(ctx.destination);
|
||||||
|
source.start(0);
|
||||||
|
} catch {
|
||||||
|
// Unlocking is best-effort.
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const stop = useCallback(() => {
|
||||||
|
const session = sessionRef.current;
|
||||||
|
if (session) {
|
||||||
|
session.cancelled = true;
|
||||||
|
session.abort.abort();
|
||||||
|
sessionRef.current = null;
|
||||||
|
}
|
||||||
|
if (audioSourceRef.current) {
|
||||||
|
try {
|
||||||
|
audioSourceRef.current.onended = null;
|
||||||
|
audioSourceRef.current.stop();
|
||||||
|
} catch {
|
||||||
|
// Already stopped
|
||||||
|
}
|
||||||
|
audioSourceRef.current = null;
|
||||||
|
}
|
||||||
|
setIsPlaying(false);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const speak = useCallback(async (text: string, options?: LocalTTSSpeakOptions): Promise<void> => {
|
||||||
|
stop();
|
||||||
|
|
||||||
|
const chunks = splitTextForSynthesis(text);
|
||||||
|
if (chunks.length === 0) {
|
||||||
|
setError('No text to speak');
|
||||||
|
options?.onError?.('No text to speak');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setError(null);
|
||||||
|
|
||||||
|
const session: PlaybackSession = { cancelled: false, abort: new AbortController() };
|
||||||
|
sessionRef.current = session;
|
||||||
|
|
||||||
|
const fetchChunk = async (chunk: string): Promise<ArrayBuffer> => {
|
||||||
|
const response = await runtimeFetch('/api/dictation/tts/speak', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
text: chunk,
|
||||||
|
...(typeof options?.speakerId === 'number' ? { speakerId: options.speakerId } : {}),
|
||||||
|
...(typeof options?.speed === 'number' ? { speed: options.speed } : {}),
|
||||||
|
}),
|
||||||
|
signal: session.abort.signal,
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||||
|
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
return (await response.blob()).arrayBuffer();
|
||||||
|
};
|
||||||
|
|
||||||
|
const playBuffer = async (arrayBuffer: ArrayBuffer): Promise<void> => {
|
||||||
|
const ctx = getAudioContext();
|
||||||
|
if (ctx.state === 'suspended') {
|
||||||
|
await ctx.resume();
|
||||||
|
}
|
||||||
|
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||||
|
if (session.cancelled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
const source = ctx.createBufferSource();
|
||||||
|
source.buffer = audioBuffer;
|
||||||
|
source.connect(ctx.destination);
|
||||||
|
audioSourceRef.current = source;
|
||||||
|
source.onended = () => {
|
||||||
|
if (audioSourceRef.current === source) {
|
||||||
|
audioSourceRef.current = null;
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
source.start(0);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
setIsPlaying(true);
|
||||||
|
options?.onStart?.();
|
||||||
|
|
||||||
|
// Pipelined: synthesize chunk N+1 while chunk N is playing.
|
||||||
|
let nextFetch: Promise<ArrayBuffer> = fetchChunk(chunks[0]);
|
||||||
|
for (let i = 0; i < chunks.length; i += 1) {
|
||||||
|
const buffer = await nextFetch;
|
||||||
|
if (session.cancelled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (i + 1 < chunks.length) {
|
||||||
|
nextFetch = fetchChunk(chunks[i + 1]);
|
||||||
|
}
|
||||||
|
await playBuffer(buffer);
|
||||||
|
if (session.cancelled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
setIsPlaying(false);
|
||||||
|
options?.onEnd?.();
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as Error).name === 'AbortError' || session.cancelled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||||
|
setError(errorMsg);
|
||||||
|
options?.onError?.(errorMsg);
|
||||||
|
setIsPlaying(false);
|
||||||
|
} finally {
|
||||||
|
if (sessionRef.current === session) {
|
||||||
|
sessionRef.current = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [stop]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
stop();
|
||||||
|
};
|
||||||
|
}, [stop]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
isPlaying,
|
||||||
|
error,
|
||||||
|
speak,
|
||||||
|
stop,
|
||||||
|
unlockAudio,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import { useCallback, useState } from 'react';
|
|||||||
import { useConfigStore } from '@/stores/useConfigStore';
|
import { useConfigStore } from '@/stores/useConfigStore';
|
||||||
import { useServerTTS } from './useServerTTS';
|
import { useServerTTS } from './useServerTTS';
|
||||||
import { useSayTTS } from './useSayTTS';
|
import { useSayTTS } from './useSayTTS';
|
||||||
|
import { useLocalTTS } from './useLocalTTS';
|
||||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||||
import { sanitizeForTTS } from '@/lib/voice/summarize';
|
import { sanitizeForTTS } from '@/lib/voice/summarize';
|
||||||
|
|
||||||
@@ -29,6 +30,7 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
|||||||
const speechPitch = useConfigStore((state) => state.speechPitch);
|
const speechPitch = useConfigStore((state) => state.speechPitch);
|
||||||
const speechVolume = useConfigStore((state) => state.speechVolume);
|
const speechVolume = useConfigStore((state) => state.speechVolume);
|
||||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||||
|
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
|
||||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||||
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
|
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
|
||||||
@@ -48,13 +50,15 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
|||||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS({
|
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS({
|
||||||
enabled: shouldCheckSayAvailability,
|
enabled: shouldCheckSayAvailability,
|
||||||
});
|
});
|
||||||
|
const { speak: speakLocalTTS, stop: stopLocalTTS } = useLocalTTS();
|
||||||
|
|
||||||
const stop = useCallback(() => {
|
const stop = useCallback(() => {
|
||||||
setIsPlaying(false);
|
setIsPlaying(false);
|
||||||
stopServerTTS();
|
stopServerTTS();
|
||||||
stopSayTTS();
|
stopSayTTS();
|
||||||
|
stopLocalTTS();
|
||||||
browserVoiceService.cancelSpeech();
|
browserVoiceService.cancelSpeech();
|
||||||
}, [stopServerTTS, stopSayTTS]);
|
}, [stopServerTTS, stopSayTTS, stopLocalTTS]);
|
||||||
|
|
||||||
const play = useCallback(async (text: string) => {
|
const play = useCallback(async (text: string) => {
|
||||||
if (!text.trim()) return;
|
if (!text.trim()) return;
|
||||||
@@ -84,6 +88,13 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
|||||||
onEnd: () => setIsPlaying(false),
|
onEnd: () => setIsPlaying(false),
|
||||||
onError: () => setIsPlaying(false),
|
onError: () => setIsPlaying(false),
|
||||||
});
|
});
|
||||||
|
} else if (voiceProvider === 'local') {
|
||||||
|
await speakLocalTTS(sanitizedText, {
|
||||||
|
speakerId: localTtsVoiceId,
|
||||||
|
speed: speechRate,
|
||||||
|
onEnd: () => setIsPlaying(false),
|
||||||
|
onError: () => setIsPlaying(false),
|
||||||
|
});
|
||||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||||
await speakSayTTS(sanitizedText, {
|
await speakSayTTS(sanitizedText, {
|
||||||
@@ -129,6 +140,8 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
|||||||
ttsInputMode,
|
ttsInputMode,
|
||||||
speakServerTTS,
|
speakServerTTS,
|
||||||
speakSayTTS,
|
speakSayTTS,
|
||||||
|
speakLocalTTS,
|
||||||
|
localTtsVoiceId,
|
||||||
stop,
|
stop,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
import { useEffect, useRef } from 'react';
|
|
||||||
import { useSessionUIStore } from '@/sync/session-ui-store';
|
|
||||||
import { useSessionPermissions, useSessionTextMessages } from '@/sync/sync-context';
|
|
||||||
import { voiceHooks, isVoiceSessionStarted } from '@/lib/voice';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Hook that syncs session events (messages, permissions) to the voice agent.
|
|
||||||
* Call this inside VoiceProvider to enable session awareness during voice.
|
|
||||||
*/
|
|
||||||
export function useVoiceContext() {
|
|
||||||
const currentSessionId = useSessionUIStore((s) => s.currentSessionId);
|
|
||||||
const messages = useSessionTextMessages(currentSessionId ?? '');
|
|
||||||
const permissions = useSessionPermissions(currentSessionId ?? '');
|
|
||||||
|
|
||||||
// Track last seen message count to only forward new messages
|
|
||||||
const lastMessageCountRef = useRef(0);
|
|
||||||
|
|
||||||
// Forward new messages to voice agent
|
|
||||||
useEffect(() => {
|
|
||||||
if (!currentSessionId || !messages || messages.length === 0 || !isVoiceSessionStarted()) return;
|
|
||||||
|
|
||||||
const currentCount = messages.length;
|
|
||||||
if (currentCount <= lastMessageCountRef.current) return;
|
|
||||||
|
|
||||||
// Get only new messages (messages since last check)
|
|
||||||
const newMessages = messages.slice(lastMessageCountRef.current);
|
|
||||||
lastMessageCountRef.current = currentCount;
|
|
||||||
|
|
||||||
const formattedMessages = newMessages.map(m => ({
|
|
||||||
role: m.role ?? '',
|
|
||||||
content: m.text,
|
|
||||||
}));
|
|
||||||
|
|
||||||
voiceHooks.onMessages(currentSessionId, formattedMessages);
|
|
||||||
}, [currentSessionId, messages]);
|
|
||||||
|
|
||||||
// Forward permission requests to voice agent
|
|
||||||
useEffect(() => {
|
|
||||||
if (!currentSessionId || !permissions || permissions.length === 0) return;
|
|
||||||
if (!isVoiceSessionStarted()) return;
|
|
||||||
|
|
||||||
const request = permissions[0];
|
|
||||||
if (!request) return;
|
|
||||||
|
|
||||||
voiceHooks.onPermissionRequested(
|
|
||||||
currentSessionId,
|
|
||||||
request.id,
|
|
||||||
request.permission,
|
|
||||||
request.metadata
|
|
||||||
);
|
|
||||||
}, [currentSessionId, permissions]);
|
|
||||||
|
|
||||||
// Reset message count when session changes
|
|
||||||
useEffect(() => {
|
|
||||||
lastMessageCountRef.current = 0;
|
|
||||||
}, [currentSessionId]);
|
|
||||||
}
|
|
||||||
@@ -178,14 +178,12 @@ export type DesktopSettings = {
|
|||||||
responseStyleEnabled?: boolean;
|
responseStyleEnabled?: boolean;
|
||||||
responseStylePreset?: 'concise' | 'detailed' | 'mentor' | 'pushback' | 'noFiller' | 'matchEnergy' | 'warmPeer' | 'custom';
|
responseStylePreset?: 'concise' | 'detailed' | 'mentor' | 'pushback' | 'noFiller' | 'matchEnergy' | 'warmPeer' | 'custom';
|
||||||
responseStyleCustomInstructions?: string;
|
responseStyleCustomInstructions?: string;
|
||||||
sttProvider?: 'browser' | 'server' | 'wasm';
|
dictationEnabled?: boolean;
|
||||||
|
sttProvider?: 'local' | 'openai-compatible';
|
||||||
sttServerUrl?: string;
|
sttServerUrl?: string;
|
||||||
sttModel?: string;
|
sttModel?: string;
|
||||||
wasmSttModel?: string;
|
sttLocalModel?: string;
|
||||||
sttLanguage?: string;
|
sttLanguage?: string;
|
||||||
sttSilenceThresholdDb?: number;
|
|
||||||
sttSilenceHoldMs?: number;
|
|
||||||
sttTranscribeOnStop?: boolean;
|
|
||||||
// Global draft welcome starters (pinned commands/skills), persisted to settings.json
|
// Global draft welcome starters (pinned commands/skills), persisted to settings.json
|
||||||
draftStarters?: DraftStarterRef[];
|
draftStarters?: DraftStarterRef[];
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,447 @@
|
|||||||
|
/**
|
||||||
|
* WebSocket client for the OpenChamber dictation endpoint (/api/dictation/ws).
|
||||||
|
*
|
||||||
|
* One shared client per app. The socket is opened lazily when a dictation
|
||||||
|
* starts and closed after an idle delay. URLs are resolved at connect time via
|
||||||
|
* the runtime URL resolver so runtime switches never leak a stale endpoint.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { getRuntimeUrlResolver } from '@/lib/runtime-url';
|
||||||
|
import { refreshRuntimeUrlAuthToken } from '@/lib/runtime-auth';
|
||||||
|
|
||||||
|
export interface DictationStartOptions {
|
||||||
|
provider?: 'local' | 'openai-compatible';
|
||||||
|
language?: string;
|
||||||
|
localModel?: string;
|
||||||
|
openaiCompatible?: {
|
||||||
|
baseUrl?: string;
|
||||||
|
model?: string;
|
||||||
|
apiKey?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DictationServerMessage {
|
||||||
|
type: string;
|
||||||
|
dictationId?: string;
|
||||||
|
ackSeq?: number;
|
||||||
|
text?: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
error?: string;
|
||||||
|
retryable?: boolean;
|
||||||
|
reasonCode?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DictationStreamError extends Error {
|
||||||
|
retryable: boolean;
|
||||||
|
reasonCode?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const createStreamError = (message: string, retryable: boolean, reasonCode?: string): DictationStreamError => {
|
||||||
|
const error = new Error(message) as DictationStreamError;
|
||||||
|
error.name = 'DictationStreamError';
|
||||||
|
error.retryable = retryable;
|
||||||
|
if (reasonCode) {
|
||||||
|
error.reasonCode = reasonCode;
|
||||||
|
}
|
||||||
|
return error;
|
||||||
|
};
|
||||||
|
|
||||||
|
const CONNECT_TIMEOUT_MS = 10000;
|
||||||
|
const START_TIMEOUT_MS = 15000;
|
||||||
|
const IDLE_CLOSE_DELAY_MS = 30000;
|
||||||
|
const DEFAULT_FINISH_TIMEOUT_MS = 30000;
|
||||||
|
|
||||||
|
type ConnectionStatusListener = (connected: boolean) => void;
|
||||||
|
type PartialListener = (dictationId: string, text: string) => void;
|
||||||
|
|
||||||
|
interface PendingStart {
|
||||||
|
resolve: () => void;
|
||||||
|
reject: (error: Error) => void;
|
||||||
|
timeout: ReturnType<typeof setTimeout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingFinish {
|
||||||
|
resolve: (result: { text: string }) => void;
|
||||||
|
reject: (error: Error) => void;
|
||||||
|
timeout: ReturnType<typeof setTimeout> | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DictationClient {
|
||||||
|
private socket: WebSocket | null = null;
|
||||||
|
private connectPromise: Promise<void> | null = null;
|
||||||
|
private idleCloseTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
private readonly pendingStarts = new Map<string, PendingStart>();
|
||||||
|
private readonly pendingFinishes = new Map<string, PendingFinish>();
|
||||||
|
private readonly connectionListeners = new Set<ConnectionStatusListener>();
|
||||||
|
private readonly partialListeners = new Set<PartialListener>();
|
||||||
|
private activeDictations = 0;
|
||||||
|
|
||||||
|
get isConnected(): boolean {
|
||||||
|
return this.socket?.readyState === WebSocket.OPEN;
|
||||||
|
}
|
||||||
|
|
||||||
|
subscribeConnectionStatus(listener: ConnectionStatusListener): () => void {
|
||||||
|
this.connectionListeners.add(listener);
|
||||||
|
return () => {
|
||||||
|
this.connectionListeners.delete(listener);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
onPartial(listener: PartialListener): () => void {
|
||||||
|
this.partialListeners.add(listener);
|
||||||
|
return () => {
|
||||||
|
this.partialListeners.delete(listener);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureConnected(): Promise<void> {
|
||||||
|
if (this.isConnected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.connectPromise) {
|
||||||
|
await this.connectPromise;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A WebSocket upgrade can't carry an Authorization header, so it
|
||||||
|
// authenticates via the oc_url_token query param. Mint/await a valid
|
||||||
|
// token BEFORE connecting — the sync getter returns "" while the token
|
||||||
|
// is unminted or inside its expiry skew, and the server would reject
|
||||||
|
// the upgrade with 401.
|
||||||
|
try {
|
||||||
|
await refreshRuntimeUrlAuthToken();
|
||||||
|
} catch {
|
||||||
|
// No auth configured (local runtime) — proceed without a token.
|
||||||
|
}
|
||||||
|
|
||||||
|
this.connectPromise = new Promise<void>((resolve, reject) => {
|
||||||
|
let settled = false;
|
||||||
|
let socket: WebSocket;
|
||||||
|
try {
|
||||||
|
const url = getRuntimeUrlResolver().websocket('/api/dictation/ws');
|
||||||
|
socket = new WebSocket(url);
|
||||||
|
} catch (error) {
|
||||||
|
this.connectPromise = null;
|
||||||
|
reject(error instanceof Error ? error : new Error(String(error)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
this.connectPromise = null;
|
||||||
|
try {
|
||||||
|
socket.close();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
reject(new Error('Dictation connection timed out'));
|
||||||
|
}
|
||||||
|
}, CONNECT_TIMEOUT_MS);
|
||||||
|
|
||||||
|
socket.onopen = () => {
|
||||||
|
// Wait for the server 'ready' frame before resolving.
|
||||||
|
};
|
||||||
|
|
||||||
|
socket.onmessage = (event) => {
|
||||||
|
let message: DictationServerMessage;
|
||||||
|
try {
|
||||||
|
message = JSON.parse(String(event.data));
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!settled && message.type === 'ready') {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timeout);
|
||||||
|
this.socket = socket;
|
||||||
|
this.connectPromise = null;
|
||||||
|
this.notifyConnection(true);
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.handleMessage(message);
|
||||||
|
};
|
||||||
|
|
||||||
|
socket.onerror = () => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timeout);
|
||||||
|
this.connectPromise = null;
|
||||||
|
reject(new Error('Dictation connection failed'));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
socket.onclose = () => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timeout);
|
||||||
|
this.connectPromise = null;
|
||||||
|
reject(new Error('Dictation connection closed'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.socket === socket) {
|
||||||
|
this.socket = null;
|
||||||
|
this.rejectAllPending(new Error('Dictation connection lost'));
|
||||||
|
this.notifyConnection(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.connectPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start a dictation stream. Resolves once the server acks the stream.
|
||||||
|
*/
|
||||||
|
async startDictationStream(
|
||||||
|
dictationId: string,
|
||||||
|
format: string,
|
||||||
|
options: DictationStartOptions,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.ensureConnected();
|
||||||
|
this.activeDictations += 1;
|
||||||
|
this.clearIdleCloseTimer();
|
||||||
|
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
this.pendingStarts.delete(dictationId);
|
||||||
|
this.releaseDictation();
|
||||||
|
reject(new Error('Dictation start timed out'));
|
||||||
|
}, START_TIMEOUT_MS);
|
||||||
|
|
||||||
|
this.pendingStarts.set(dictationId, {
|
||||||
|
resolve: () => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
reject: (error) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
this.releaseDictation();
|
||||||
|
reject(error);
|
||||||
|
},
|
||||||
|
timeout });
|
||||||
|
|
||||||
|
if (!this.send({ type: 'start', dictationId, format, options })) {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
this.pendingStarts.delete(dictationId);
|
||||||
|
this.releaseDictation();
|
||||||
|
reject(new Error('Dictation connection lost'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
sendDictationStreamChunk(dictationId: string, seq: number, audioBase64: string): boolean {
|
||||||
|
return this.send({ type: 'chunk', dictationId, seq, audio: audioBase64 });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finish a dictation stream. Resolves with the final transcript.
|
||||||
|
*/
|
||||||
|
finishDictationStream(dictationId: string, finalSeq: number): Promise<{ text: string }> {
|
||||||
|
return new Promise<{ text: string }>((resolve, reject) => {
|
||||||
|
const pending: PendingFinish = {
|
||||||
|
resolve: (result) => {
|
||||||
|
if (pending.timeout) {
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
}
|
||||||
|
this.releaseDictation();
|
||||||
|
resolve(result);
|
||||||
|
},
|
||||||
|
reject: (error) => {
|
||||||
|
if (pending.timeout) {
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
}
|
||||||
|
this.releaseDictation();
|
||||||
|
reject(error);
|
||||||
|
},
|
||||||
|
timeout: null,
|
||||||
|
};
|
||||||
|
pending.timeout = setTimeout(() => {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
this.releaseDictation();
|
||||||
|
reject(new Error('Timed out waiting for transcription'));
|
||||||
|
}, DEFAULT_FINISH_TIMEOUT_MS);
|
||||||
|
|
||||||
|
this.pendingFinishes.set(dictationId, pending);
|
||||||
|
|
||||||
|
if (!this.send({ type: 'finish', dictationId, finalSeq })) {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
pending.reject(new Error('Dictation connection lost'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
cancelDictationStream(dictationId: string): void {
|
||||||
|
this.send({ type: 'cancel', dictationId });
|
||||||
|
const start = this.pendingStarts.get(dictationId);
|
||||||
|
if (start) {
|
||||||
|
this.pendingStarts.delete(dictationId);
|
||||||
|
start.reject(new Error('Dictation cancelled'));
|
||||||
|
}
|
||||||
|
const finish = this.pendingFinishes.get(dictationId);
|
||||||
|
if (finish) {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
finish.reject(new Error('Dictation cancelled'));
|
||||||
|
}
|
||||||
|
this.releaseDictation();
|
||||||
|
this.scheduleIdleCloseIfReady();
|
||||||
|
}
|
||||||
|
|
||||||
|
private handleMessage(message: DictationServerMessage): void {
|
||||||
|
const dictationId = message.dictationId;
|
||||||
|
if (!dictationId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (message.type) {
|
||||||
|
case 'ack': {
|
||||||
|
const pendingStart = this.pendingStarts.get(dictationId);
|
||||||
|
if (pendingStart) {
|
||||||
|
this.pendingStarts.delete(dictationId);
|
||||||
|
pendingStart.resolve();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'partial': {
|
||||||
|
for (const listener of this.partialListeners) {
|
||||||
|
listener(dictationId, message.text ?? '');
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'finish_accepted': {
|
||||||
|
const pendingFinish = this.pendingFinishes.get(dictationId);
|
||||||
|
if (pendingFinish && typeof message.timeoutMs === 'number') {
|
||||||
|
if (pendingFinish.timeout) {
|
||||||
|
clearTimeout(pendingFinish.timeout);
|
||||||
|
}
|
||||||
|
pendingFinish.timeout = setTimeout(() => {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
pendingFinish.reject(new Error('Timed out waiting for transcription'));
|
||||||
|
}, message.timeoutMs + 5000);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'final': {
|
||||||
|
const pendingFinish = this.pendingFinishes.get(dictationId);
|
||||||
|
if (pendingFinish) {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
pendingFinish.resolve({ text: message.text ?? '' });
|
||||||
|
}
|
||||||
|
this.scheduleIdleCloseIfReady();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'error': {
|
||||||
|
const error = createStreamError(
|
||||||
|
message.error || 'Dictation failed',
|
||||||
|
message.retryable !== false,
|
||||||
|
message.reasonCode,
|
||||||
|
);
|
||||||
|
const pendingStart = this.pendingStarts.get(dictationId);
|
||||||
|
if (pendingStart) {
|
||||||
|
this.pendingStarts.delete(dictationId);
|
||||||
|
pendingStart.reject(error);
|
||||||
|
}
|
||||||
|
const pendingFinish = this.pendingFinishes.get(dictationId);
|
||||||
|
if (pendingFinish) {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
pendingFinish.reject(error);
|
||||||
|
}
|
||||||
|
this.scheduleIdleCloseIfReady();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private send(message: object): boolean {
|
||||||
|
if (!this.isConnected || !this.socket) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
this.socket.send(JSON.stringify(message));
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private notifyConnection(connected: boolean): void {
|
||||||
|
for (const listener of this.connectionListeners) {
|
||||||
|
listener(connected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private rejectAllPending(error: Error): void {
|
||||||
|
for (const [dictationId, pending] of this.pendingStarts) {
|
||||||
|
this.pendingStarts.delete(dictationId);
|
||||||
|
pending.reject(error);
|
||||||
|
}
|
||||||
|
for (const [dictationId, pending] of this.pendingFinishes) {
|
||||||
|
this.pendingFinishes.delete(dictationId);
|
||||||
|
pending.reject(error);
|
||||||
|
}
|
||||||
|
this.activeDictations = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private releaseDictation(): void {
|
||||||
|
this.activeDictations = Math.max(0, this.activeDictations - 1);
|
||||||
|
this.scheduleIdleCloseIfReady();
|
||||||
|
}
|
||||||
|
|
||||||
|
private scheduleIdleCloseIfReady(): void {
|
||||||
|
if (this.activeDictations > 0 || this.pendingFinishes.size > 0 || this.pendingStarts.size > 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.clearIdleCloseTimer();
|
||||||
|
this.idleCloseTimer = setTimeout(() => {
|
||||||
|
if (this.activeDictations === 0 && this.pendingFinishes.size === 0 && this.pendingStarts.size === 0) {
|
||||||
|
const socket = this.socket;
|
||||||
|
this.socket = null;
|
||||||
|
if (socket) {
|
||||||
|
try {
|
||||||
|
socket.close(1000, 'idle');
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
this.notifyConnection(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, IDLE_CLOSE_DELAY_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runtime switch: close the socket and fail all in-flight dictations so
|
||||||
|
* nothing keeps streaming to the previous runtime.
|
||||||
|
*/
|
||||||
|
cancelAllForRuntimeSwitch(): void {
|
||||||
|
this.clearIdleCloseTimer();
|
||||||
|
const socket = this.socket;
|
||||||
|
this.socket = null;
|
||||||
|
this.connectPromise = null;
|
||||||
|
this.rejectAllPending(new Error('Runtime changed'));
|
||||||
|
if (socket) {
|
||||||
|
try {
|
||||||
|
socket.close(1000, 'runtime switch');
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
this.notifyConnection(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearIdleCloseTimer(): void {
|
||||||
|
if (this.idleCloseTimer) {
|
||||||
|
clearTimeout(this.idleCloseTimer);
|
||||||
|
this.idleCloseTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const dictationClient = new DictationClient();
|
||||||
|
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
window.addEventListener('openchamber:runtime-endpoint-changed', () => {
|
||||||
|
// Drop the socket so the next dictation reconnects to the new runtime.
|
||||||
|
dictationClient.cancelAllForRuntimeSwitch();
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
/**
|
||||||
|
* Small, non-React state machine for dictation streaming.
|
||||||
|
*
|
||||||
|
* Responsibilities:
|
||||||
|
* - Maintain an ordered buffer of base64 PCM segments
|
||||||
|
* - Start/restart a dictation stream (dictationId)
|
||||||
|
* - Send missing segments (seq) when connected
|
||||||
|
* - Finish/cancel the stream
|
||||||
|
*
|
||||||
|
* Segments are retained until the dictation completes, which enables replay
|
||||||
|
* after a connection drop (`resetStreamForReplay()` + `finish()`).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { DictationClient, DictationStartOptions } from './dictation-client';
|
||||||
|
|
||||||
|
const MAX_CHUNKS_PER_FLUSH_TURN = 128;
|
||||||
|
|
||||||
|
const PCM_DICTATION_FORMAT = 'audio/pcm;rate=16000;bits=16';
|
||||||
|
|
||||||
|
const waitForNextFlushTurn = (): Promise<void> => new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
|
||||||
|
const createDictationIdDefault = (): string => {
|
||||||
|
const rand = Math.random().toString(36).slice(2, 10);
|
||||||
|
return `dic_${Date.now().toString(16)}${rand}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface DictationFinishResult {
|
||||||
|
dictationId: string;
|
||||||
|
text: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DictationStreamSender {
|
||||||
|
private readonly client: DictationClient;
|
||||||
|
private readonly format: string;
|
||||||
|
private readonly createDictationId: () => string;
|
||||||
|
private getStartOptions: () => DictationStartOptions;
|
||||||
|
|
||||||
|
private dictationId: string | null = null;
|
||||||
|
private sendSeq = 0;
|
||||||
|
private segments: string[] = [];
|
||||||
|
private streamReady = false;
|
||||||
|
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
private drainWaiters: Array<() => void> = [];
|
||||||
|
|
||||||
|
private startGeneration = 0;
|
||||||
|
private startPromise: Promise<void> | null = null;
|
||||||
|
|
||||||
|
constructor(params: {
|
||||||
|
client: DictationClient;
|
||||||
|
getStartOptions: () => DictationStartOptions;
|
||||||
|
format?: string;
|
||||||
|
createDictationId?: () => string;
|
||||||
|
}) {
|
||||||
|
this.client = params.client;
|
||||||
|
this.format = params.format ?? PCM_DICTATION_FORMAT;
|
||||||
|
this.getStartOptions = params.getStartOptions;
|
||||||
|
this.createDictationId = params.createDictationId ?? createDictationIdDefault;
|
||||||
|
}
|
||||||
|
|
||||||
|
getDictationId(): string | null {
|
||||||
|
return this.dictationId;
|
||||||
|
}
|
||||||
|
|
||||||
|
getFinalSeq(): number {
|
||||||
|
return this.segments.length - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
hasSegments(): boolean {
|
||||||
|
return this.segments.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
clearAll(): void {
|
||||||
|
this.clearScheduledFlush();
|
||||||
|
this.dictationId = null;
|
||||||
|
this.sendSeq = 0;
|
||||||
|
this.segments = [];
|
||||||
|
this.streamReady = false;
|
||||||
|
this.startPromise = null;
|
||||||
|
this.startGeneration += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
resetStreamForReplay(): void {
|
||||||
|
this.clearScheduledFlush();
|
||||||
|
this.dictationId = null;
|
||||||
|
this.sendSeq = 0;
|
||||||
|
this.streamReady = false;
|
||||||
|
this.startPromise = null;
|
||||||
|
this.startGeneration += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
enqueueSegment(base64Pcm: string): void {
|
||||||
|
this.segments.push(base64Pcm);
|
||||||
|
|
||||||
|
if (!this.client.isConnected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.dictationId) {
|
||||||
|
if (!this.startPromise) {
|
||||||
|
void this.restartStream().catch(() => {
|
||||||
|
// Start failures surface through finish(); segments are retained.
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
flush(): number {
|
||||||
|
const dictationId = this.dictationId;
|
||||||
|
if (!this.client.isConnected || !dictationId || !this.streamReady) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sent = 0;
|
||||||
|
while (this.sendSeq < this.segments.length && sent < MAX_CHUNKS_PER_FLUSH_TURN) {
|
||||||
|
const seq = this.sendSeq;
|
||||||
|
const audio = this.segments[seq];
|
||||||
|
if (!this.client.sendDictationStreamChunk(dictationId, seq, audio)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
this.sendSeq = seq + 1;
|
||||||
|
sent += 1;
|
||||||
|
}
|
||||||
|
if (this.hasPendingSegments()) {
|
||||||
|
this.scheduleFlush();
|
||||||
|
} else {
|
||||||
|
this.resolveDrainWaiters();
|
||||||
|
}
|
||||||
|
return sent;
|
||||||
|
}
|
||||||
|
|
||||||
|
async restartStream(): Promise<void> {
|
||||||
|
this.startGeneration += 1;
|
||||||
|
const generation = this.startGeneration;
|
||||||
|
|
||||||
|
const dictationId = this.createDictationId();
|
||||||
|
this.dictationId = dictationId;
|
||||||
|
this.sendSeq = 0;
|
||||||
|
this.streamReady = false;
|
||||||
|
|
||||||
|
const start = (async () => {
|
||||||
|
await this.client.startDictationStream(dictationId, this.format, this.getStartOptions());
|
||||||
|
if (this.startGeneration !== generation) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.dictationId !== dictationId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.streamReady = true;
|
||||||
|
this.flush();
|
||||||
|
})()
|
||||||
|
.catch((error) => {
|
||||||
|
// Keep segments for retry, but clear the stream so finish can error cleanly.
|
||||||
|
if (this.startGeneration === generation && this.dictationId === dictationId) {
|
||||||
|
this.dictationId = null;
|
||||||
|
this.streamReady = false;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (this.startPromise === start) {
|
||||||
|
this.startPromise = null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.startPromise = start;
|
||||||
|
await start;
|
||||||
|
}
|
||||||
|
|
||||||
|
async finish(finalSeq: number): Promise<DictationFinishResult> {
|
||||||
|
if (!this.dictationId) {
|
||||||
|
await this.restartStream();
|
||||||
|
}
|
||||||
|
if (this.startPromise) {
|
||||||
|
await this.startPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dictationId = this.dictationId;
|
||||||
|
if (!dictationId || !this.streamReady) {
|
||||||
|
throw new Error('Failed to start dictation stream');
|
||||||
|
}
|
||||||
|
|
||||||
|
this.flush();
|
||||||
|
await this.waitForFlushDrain();
|
||||||
|
const result = await this.client.finishDictationStream(dictationId, finalSeq);
|
||||||
|
return { dictationId, text: result.text };
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
const dictationId = this.dictationId;
|
||||||
|
if (this.client.isConnected && dictationId) {
|
||||||
|
this.client.cancelDictationStream(dictationId);
|
||||||
|
}
|
||||||
|
this.resetStreamForReplay();
|
||||||
|
}
|
||||||
|
|
||||||
|
private hasPendingSegments(): boolean {
|
||||||
|
return this.sendSeq < this.segments.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
private scheduleFlush(): void {
|
||||||
|
if (this.flushTimer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.flushTimer = setTimeout(() => {
|
||||||
|
this.flushTimer = null;
|
||||||
|
this.flush();
|
||||||
|
}, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearScheduledFlush(): void {
|
||||||
|
if (!this.flushTimer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
clearTimeout(this.flushTimer);
|
||||||
|
this.flushTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async waitForFlushDrain(): Promise<void> {
|
||||||
|
while (this.hasPendingSegments()) {
|
||||||
|
if (!this.client.isConnected || !this.dictationId || !this.streamReady) {
|
||||||
|
throw new Error('Failed to flush dictation stream');
|
||||||
|
}
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
this.drainWaiters.push(resolve);
|
||||||
|
});
|
||||||
|
await waitForNextFlushTurn();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveDrainWaiters(): void {
|
||||||
|
const waiters = this.drainWaiters;
|
||||||
|
this.drainWaiters = [];
|
||||||
|
for (const resolve of waiters) {
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,301 @@
|
|||||||
|
/**
|
||||||
|
* Microphone capture for dictation.
|
||||||
|
*
|
||||||
|
* Captures mono audio via getUserMedia, taps it with a ScriptProcessorNode
|
||||||
|
* (universally supported, including iOS WKWebView), resamples Float32 to
|
||||||
|
* 16 kHz PCM16LE, and emits ~1-second base64 chunks plus a normalized RMS
|
||||||
|
* volume for the level meter.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||||
|
|
||||||
|
export interface DictationAudioSourceConfig {
|
||||||
|
onPcmSegment: (base64Pcm: string) => void;
|
||||||
|
onError?: (error: Error) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DictationAudioSource {
|
||||||
|
start: () => Promise<void>;
|
||||||
|
stop: () => Promise<void>;
|
||||||
|
volume: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const OUTPUT_RATE = 16000;
|
||||||
|
const CHUNK_SAMPLES = OUTPUT_RATE; // ~1s per chunk
|
||||||
|
|
||||||
|
const getAudioContextCtor = (): typeof AudioContext | null => {
|
||||||
|
if (typeof window === 'undefined') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const win = window as typeof window & { webkitAudioContext?: typeof AudioContext };
|
||||||
|
return win.AudioContext || win.webkitAudioContext || null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const floatToInt16 = (sample: number): number => {
|
||||||
|
const clamped = Math.max(-1, Math.min(1, sample));
|
||||||
|
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||||
|
};
|
||||||
|
|
||||||
|
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
|
||||||
|
if (input.length === 0) {
|
||||||
|
return new Int16Array(0);
|
||||||
|
}
|
||||||
|
if (inputRate === outputRate) {
|
||||||
|
const out = new Int16Array(input.length);
|
||||||
|
for (let i = 0; i < input.length; i++) {
|
||||||
|
out[i] = floatToInt16(input[i]);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ratio = inputRate / outputRate;
|
||||||
|
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||||
|
const out = new Int16Array(outputLength);
|
||||||
|
for (let i = 0; i < outputLength; i++) {
|
||||||
|
const sourceIndex = i * ratio;
|
||||||
|
const i0 = Math.floor(sourceIndex);
|
||||||
|
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||||
|
const frac = sourceIndex - i0;
|
||||||
|
out[i] = floatToInt16(input[i0] * (1 - frac) + input[i1] * frac);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
|
||||||
|
const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => {
|
||||||
|
if (a.length === 0) {
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
if (b.length === 0) {
|
||||||
|
return a;
|
||||||
|
}
|
||||||
|
const out = new Int16Array(a.length + b.length);
|
||||||
|
out.set(a, 0);
|
||||||
|
out.set(b, a.length);
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
|
||||||
|
const int16ToBase64 = (pcm: Int16Array): string => {
|
||||||
|
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||||
|
let binary = '';
|
||||||
|
for (let i = 0; i < bytes.length; i++) {
|
||||||
|
binary += String.fromCharCode(bytes[i]);
|
||||||
|
}
|
||||||
|
return btoa(binary);
|
||||||
|
};
|
||||||
|
|
||||||
|
interface CaptureGraph {
|
||||||
|
stream: MediaStream | null;
|
||||||
|
context: AudioContext | null;
|
||||||
|
source: MediaStreamAudioSourceNode | null;
|
||||||
|
processor: ScriptProcessorNode | null;
|
||||||
|
gain: GainNode | null;
|
||||||
|
pending: Int16Array;
|
||||||
|
started: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const emptyGraph = (): CaptureGraph => ({
|
||||||
|
stream: null,
|
||||||
|
context: null,
|
||||||
|
source: null,
|
||||||
|
processor: null,
|
||||||
|
gain: null,
|
||||||
|
pending: new Int16Array(0),
|
||||||
|
started: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const safeDisconnect = (node: AudioNode | null): void => {
|
||||||
|
if (!node) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
node.disconnect();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isDictationCaptureSupported = (): boolean => {
|
||||||
|
if (typeof navigator === 'undefined' || typeof window === 'undefined') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return getAudioContextCtor() !== null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||||
|
const [volume, setVolume] = useState(0);
|
||||||
|
|
||||||
|
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||||
|
const onErrorRef = useRef(config.onError);
|
||||||
|
useEffect(() => {
|
||||||
|
onPcmSegmentRef.current = config.onPcmSegment;
|
||||||
|
onErrorRef.current = config.onError;
|
||||||
|
}, [config.onPcmSegment, config.onError]);
|
||||||
|
|
||||||
|
const graphRef = useRef<CaptureGraph>(emptyGraph());
|
||||||
|
|
||||||
|
const start = useCallback(async () => {
|
||||||
|
if (graphRef.current.started) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
typeof navigator === 'undefined' ||
|
||||||
|
!navigator.mediaDevices ||
|
||||||
|
typeof navigator.mediaDevices.getUserMedia !== 'function'
|
||||||
|
) {
|
||||||
|
throw new Error('Microphone capture is not supported in this environment');
|
||||||
|
}
|
||||||
|
|
||||||
|
const AudioContextCtor = getAudioContextCtor();
|
||||||
|
if (!AudioContextCtor) {
|
||||||
|
throw new Error('AudioContext unavailable');
|
||||||
|
}
|
||||||
|
|
||||||
|
const stream = await navigator.mediaDevices.getUserMedia({
|
||||||
|
audio: {
|
||||||
|
channelCount: 1,
|
||||||
|
noiseSuppression: true,
|
||||||
|
echoCancellation: true,
|
||||||
|
autoGainControl: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const context = new AudioContextCtor();
|
||||||
|
try {
|
||||||
|
if (context.state === 'suspended') {
|
||||||
|
await context.resume().catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
const source = context.createMediaStreamSource(stream);
|
||||||
|
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||||
|
const gain = context.createGain();
|
||||||
|
gain.gain.value = 0;
|
||||||
|
|
||||||
|
graphRef.current = {
|
||||||
|
stream,
|
||||||
|
context,
|
||||||
|
source,
|
||||||
|
processor,
|
||||||
|
gain,
|
||||||
|
pending: new Int16Array(0),
|
||||||
|
started: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
processor.onaudioprocess = (event) => {
|
||||||
|
const graph = graphRef.current;
|
||||||
|
if (!graph.started) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const input = event.inputBuffer.getChannelData(0);
|
||||||
|
|
||||||
|
let sumSquares = 0;
|
||||||
|
for (let i = 0; i < input.length; i++) {
|
||||||
|
sumSquares += input[i] * input[i];
|
||||||
|
}
|
||||||
|
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||||
|
setVolume(Math.min(1, Math.max(0, rms * 2)));
|
||||||
|
|
||||||
|
const next = resampleToPcm16(input, context.sampleRate, OUTPUT_RATE);
|
||||||
|
graph.pending = concatInt16(graph.pending, next);
|
||||||
|
|
||||||
|
while (graph.pending.length >= CHUNK_SAMPLES) {
|
||||||
|
const chunk = graph.pending.slice(0, CHUNK_SAMPLES);
|
||||||
|
graph.pending = graph.pending.slice(CHUNK_SAMPLES);
|
||||||
|
onPcmSegmentRef.current(int16ToBase64(chunk));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
source.connect(processor);
|
||||||
|
processor.connect(gain);
|
||||||
|
gain.connect(context.destination);
|
||||||
|
} catch (error) {
|
||||||
|
stream.getTracks().forEach((track) => {
|
||||||
|
try {
|
||||||
|
track.stop();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await context.close();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
graphRef.current = emptyGraph();
|
||||||
|
throw error instanceof Error ? error : new Error(String(error));
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const stop = useCallback(async () => {
|
||||||
|
const graph = graphRef.current;
|
||||||
|
graph.started = false;
|
||||||
|
setVolume(0);
|
||||||
|
|
||||||
|
if (graph.processor) {
|
||||||
|
try {
|
||||||
|
graph.processor.onaudioprocess = null;
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
}
|
||||||
|
safeDisconnect(graph.processor);
|
||||||
|
safeDisconnect(graph.source);
|
||||||
|
safeDisconnect(graph.gain);
|
||||||
|
if (graph.stream) {
|
||||||
|
graph.stream.getTracks().forEach((track) => {
|
||||||
|
try {
|
||||||
|
track.stop();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const pending = graph.pending;
|
||||||
|
graph.pending = new Int16Array(0);
|
||||||
|
if (pending.length > 0) {
|
||||||
|
onPcmSegmentRef.current(int16ToBase64(pending));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (graph.context) {
|
||||||
|
try {
|
||||||
|
await graph.context.close();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new capture may have started while the old context was closing;
|
||||||
|
// only clear the ref if it still points at the graph we tore down.
|
||||||
|
if (graphRef.current === graph) {
|
||||||
|
graphRef.current = emptyGraph();
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
void stop().catch((err) => {
|
||||||
|
onErrorRef.current?.(err instanceof Error ? err : new Error(String(err)));
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}, [stop]);
|
||||||
|
|
||||||
|
return useMemo(
|
||||||
|
() => ({
|
||||||
|
start: async () => {
|
||||||
|
try {
|
||||||
|
await start();
|
||||||
|
} catch (err) {
|
||||||
|
const normalized = err instanceof Error ? err : new Error(String(err));
|
||||||
|
onErrorRef.current?.(normalized);
|
||||||
|
throw normalized;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
stop,
|
||||||
|
volume,
|
||||||
|
}),
|
||||||
|
[start, stop, volume],
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1518,16 +1518,33 @@ export const settingsDict = {
|
|||||||
'settings.notifications.page.toast.backgroundDisabled': 'Background notifications disabled',
|
'settings.notifications.page.toast.backgroundDisabled': 'Background notifications disabled',
|
||||||
'settings.notifications.page.testNotification.title': 'Test Notification',
|
'settings.notifications.page.testNotification.title': 'Test Notification',
|
||||||
'settings.notifications.page.testNotification.body': 'This is a test notification from OpenChamber.',
|
'settings.notifications.page.testNotification.body': 'This is a test notification from OpenChamber.',
|
||||||
'settings.voice.page.section.voiceSetup': 'Voice Setup',
|
|
||||||
'settings.voice.page.section.speechRecognition': 'Speech Recognition',
|
'settings.voice.page.section.speechRecognition': 'Speech Recognition',
|
||||||
|
'settings.voice.page.field.enableVoiceInput': 'Enable voice input',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': 'Enable voice input (dictation)',
|
||||||
'settings.voice.page.section.playbackAndSummary': 'Playback',
|
'settings.voice.page.section.playbackAndSummary': 'Playback',
|
||||||
'settings.voice.page.field.enableVoiceModeAria': 'Enable voice mode',
|
|
||||||
'settings.voice.page.field.enableVoiceMode': 'Enable Voice Mode',
|
|
||||||
'settings.voice.page.field.provider': 'Provider',
|
'settings.voice.page.field.provider': 'Provider',
|
||||||
'settings.voice.page.provider.browser': 'Browser',
|
'settings.voice.page.provider.browser': 'Browser',
|
||||||
'settings.voice.page.provider.custom': 'Custom',
|
'settings.voice.page.provider.custom': 'Custom',
|
||||||
'settings.voice.page.provider.say': 'Say',
|
'settings.voice.page.provider.say': 'Say',
|
||||||
'settings.voice.page.provider.server': 'Server',
|
'settings.voice.page.provider.server': 'Server',
|
||||||
|
'settings.voice.page.provider.local': 'Local',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': 'On-device transcription on the OpenChamber server. Models download automatically; no API key needed.',
|
||||||
|
'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro, English). The model downloads automatically; no API key needed.',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (English)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 European languages)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingual)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (multilingual)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': 'Best for English',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': 'Best for multilingual',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': 'Accuracy',
|
||||||
|
'settings.voice.page.stt.meta.speed': 'Speed',
|
||||||
|
'settings.voice.page.stt.modelInstalled': 'Model installed',
|
||||||
|
'settings.voice.page.stt.modelDownloading': 'Downloading model...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': 'Downloading model... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': 'Model not downloaded',
|
||||||
|
'settings.voice.page.stt.modelDownload': 'Download',
|
||||||
|
'settings.voice.page.stt.modelDelete': 'Delete model',
|
||||||
|
'settings.voice.page.stt.modelRetry': 'Retry',
|
||||||
'settings.voice.page.provider.wasm': 'Local',
|
'settings.voice.page.provider.wasm': 'Local',
|
||||||
'settings.voice.page.stt.wasmModel': 'Whisper Model',
|
'settings.voice.page.stt.wasmModel': 'Whisper Model',
|
||||||
'settings.voice.page.stt.wasmLoaded': 'Model loaded, ready',
|
'settings.voice.page.stt.wasmLoaded': 'Model loaded, ready',
|
||||||
|
|||||||
@@ -1873,6 +1873,18 @@ export const dict = {
|
|||||||
'chat.chatInput.placeholder.chat': '@ for files/agents; / for commands and skills; ! for shell; # for snippets',
|
'chat.chatInput.placeholder.chat': '@ for files/agents; / for commands and skills; ! for shell; # for snippets',
|
||||||
'chat.chatInput.placeholder.chatCompact': 'Use @ / ! # for helpers',
|
'chat.chatInput.placeholder.chatCompact': 'Use @ / ! # for helpers',
|
||||||
'chat.chatInput.placeholder.selectSession': 'Select or create a session to start chatting',
|
'chat.chatInput.placeholder.selectSession': 'Select or create a session to start chatting',
|
||||||
|
'chat.dictation.start': 'Start dictation',
|
||||||
|
'chat.dictation.overlayAria': 'Dictation',
|
||||||
|
'chat.dictation.downloadingModel': 'Downloading speech model...',
|
||||||
|
'chat.dictation.downloadingModelProgress': 'Downloading speech model... {percent}%',
|
||||||
|
'chat.dictation.listening': 'Listening...',
|
||||||
|
'chat.dictation.processing': 'Transcribing...',
|
||||||
|
'chat.dictation.failed': 'Transcription failed',
|
||||||
|
'chat.dictation.cancel': 'Discard dictation',
|
||||||
|
'chat.dictation.insert': 'Insert transcript',
|
||||||
|
'chat.dictation.insertAndSend': 'Insert and send',
|
||||||
|
'chat.dictation.retry': 'Retry transcription',
|
||||||
|
'chat.dictation.discard': 'Discard recording',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ Add new snippet',
|
'chat.snippetAutocomplete.action.addNew': '+ Add new snippet',
|
||||||
'chat.snippetAutocomplete.empty': 'No snippets found',
|
'chat.snippetAutocomplete.empty': 'No snippets found',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ navigate • Enter select • Esc close',
|
'chat.snippetAutocomplete.footer': '↑↓ navigate • Enter select • Esc close',
|
||||||
|
|||||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
|||||||
"settings.notifications.page.toast.backgroundDisabled": "Notificaciones de fondo deshabilitadas",
|
"settings.notifications.page.toast.backgroundDisabled": "Notificaciones de fondo deshabilitadas",
|
||||||
"settings.notifications.page.testNotification.title": "Notificación de prueba",
|
"settings.notifications.page.testNotification.title": "Notificación de prueba",
|
||||||
"settings.notifications.page.testNotification.body": "Esta es una notificación de prueba de OpenChamber.",
|
"settings.notifications.page.testNotification.body": "Esta es una notificación de prueba de OpenChamber.",
|
||||||
"settings.voice.page.section.voiceSetup": "Configuración de voz",
|
|
||||||
"settings.voice.page.section.speechRecognition": "Reconocimiento de voz",
|
"settings.voice.page.section.speechRecognition": "Reconocimiento de voz",
|
||||||
|
"settings.voice.page.field.enableVoiceInput": "Habilitar entrada de voz",
|
||||||
|
"settings.voice.page.field.enableVoiceInputAria": "Habilitar entrada de voz (dictado)",
|
||||||
"settings.voice.page.section.playbackAndSummary": "Reproducción",
|
"settings.voice.page.section.playbackAndSummary": "Reproducción",
|
||||||
"settings.voice.page.field.enableVoiceModeAria": "Habilitar modo de voz",
|
|
||||||
"settings.voice.page.field.enableVoiceMode": "Habilitar modo de voz",
|
|
||||||
"settings.voice.page.field.provider": "Proveedor",
|
"settings.voice.page.field.provider": "Proveedor",
|
||||||
"settings.voice.page.provider.browser": "Navegador",
|
"settings.voice.page.provider.browser": "Navegador",
|
||||||
"settings.voice.page.provider.custom": "Personalizado",
|
"settings.voice.page.provider.custom": "Personalizado",
|
||||||
"settings.voice.page.provider.say": "Decir",
|
"settings.voice.page.provider.say": "Decir",
|
||||||
"settings.voice.page.provider.server": "Servidor",
|
"settings.voice.page.provider.server": "Servidor",
|
||||||
|
"settings.voice.page.provider.local": "Local",
|
||||||
|
"settings.voice.page.tooltip.sttLocal": "Transcripción local en el servidor de OpenChamber. Los modelos se descargan automáticamente; no se necesita clave de API.",
|
||||||
|
"settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro, inglés). El modelo se descarga automáticamente; no se necesita clave de API.",
|
||||||
|
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglés)",
|
||||||
|
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeos)",
|
||||||
|
"settings.voice.page.stt.model.whisperBase": "Whisper base (multilingüe)",
|
||||||
|
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (multilingüe)",
|
||||||
|
"settings.voice.page.stt.badge.bestForEnglish": "Mejor para inglés",
|
||||||
|
"settings.voice.page.stt.badge.bestForMultilingual": "Mejor multilingüe",
|
||||||
|
"settings.voice.page.stt.meta.accuracy": "Precisión",
|
||||||
|
"settings.voice.page.stt.meta.speed": "Velocidad",
|
||||||
|
"settings.voice.page.stt.modelInstalled": "Modelo instalado",
|
||||||
|
"settings.voice.page.stt.modelDownloading": "Descargando modelo...",
|
||||||
|
"settings.voice.page.stt.modelDownloadingProgress": "Descargando modelo... {percent}%",
|
||||||
|
"settings.voice.page.stt.modelNotInstalled": "Modelo no descargado",
|
||||||
|
"settings.voice.page.stt.modelDownload": "Descargar",
|
||||||
|
"settings.voice.page.stt.modelDelete": "Eliminar modelo",
|
||||||
|
"settings.voice.page.stt.modelRetry": "Reintentar",
|
||||||
"settings.voice.page.provider.wasm": "Local",
|
"settings.voice.page.provider.wasm": "Local",
|
||||||
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
|
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
|
||||||
"settings.voice.page.stt.wasmLoaded": "Modelo cargado",
|
"settings.voice.page.stt.wasmLoaded": "Modelo cargado",
|
||||||
|
|||||||
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'diffView.reviewDialog.actions.starting': 'Iniciando...',
|
'diffView.reviewDialog.actions.starting': 'Iniciando...',
|
||||||
'diffView.reviewDialog.toast.noSessionDirectory': 'El directorio de la sesión no está disponible',
|
'diffView.reviewDialog.toast.noSessionDirectory': 'El directorio de la sesión no está disponible',
|
||||||
'diffView.reviewDialog.toast.startFailed': 'No se pudo iniciar el flujo de revisión',
|
'diffView.reviewDialog.toast.startFailed': 'No se pudo iniciar el flujo de revisión',
|
||||||
|
'chat.dictation.start': 'Iniciar dictado',
|
||||||
|
'chat.dictation.overlayAria': 'Dictado',
|
||||||
|
'chat.dictation.downloadingModel': 'Descargando el modelo de voz...',
|
||||||
|
'chat.dictation.downloadingModelProgress': 'Descargando el modelo de voz... {percent}%',
|
||||||
|
'chat.dictation.listening': 'Escuchando...',
|
||||||
|
'chat.dictation.processing': 'Transcribiendo...',
|
||||||
|
'chat.dictation.failed': 'La transcripción falló',
|
||||||
|
'chat.dictation.cancel': 'Descartar dictado',
|
||||||
|
'chat.dictation.insert': 'Insertar transcripción',
|
||||||
|
'chat.dictation.insertAndSend': 'Insertar y enviar',
|
||||||
|
'chat.dictation.retry': 'Reintentar transcripción',
|
||||||
|
'chat.dictation.discard': 'Descartar grabación',
|
||||||
'chat.history.loadOlder': 'Cargar mensajes anteriores',
|
'chat.history.loadOlder': 'Cargar mensajes anteriores',
|
||||||
'chat.autoReview.title': 'El ciclo de revisión de código está en curso',
|
'chat.autoReview.title': 'El ciclo de revisión de código está en curso',
|
||||||
'chat.autoReview.status.waitingForReviewer': 'Esperando al revisor',
|
'chat.autoReview.status.waitingForReviewer': 'Esperando al revisor',
|
||||||
|
|||||||
@@ -1468,16 +1468,33 @@ export const settingsDict = {
|
|||||||
'settings.notifications.page.toast.backgroundDisabled': 'Notifications en arrière-plan désactivées',
|
'settings.notifications.page.toast.backgroundDisabled': 'Notifications en arrière-plan désactivées',
|
||||||
'settings.notifications.page.testNotification.title': 'Avis de test',
|
'settings.notifications.page.testNotification.title': 'Avis de test',
|
||||||
'settings.notifications.page.testNotification.body': 'Il s\'agit d\'une notification de test de OpenChamber.',
|
'settings.notifications.page.testNotification.body': 'Il s\'agit d\'une notification de test de OpenChamber.',
|
||||||
'settings.voice.page.section.voiceSetup': 'Configuration vocale',
|
|
||||||
'settings.voice.page.section.speechRecognition': 'Reconnaissance vocale',
|
'settings.voice.page.section.speechRecognition': 'Reconnaissance vocale',
|
||||||
|
'settings.voice.page.field.enableVoiceInput': 'Activer la saisie vocale',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': 'Activer la saisie vocale (dictée)',
|
||||||
'settings.voice.page.section.playbackAndSummary': 'Lecture',
|
'settings.voice.page.section.playbackAndSummary': 'Lecture',
|
||||||
'settings.voice.page.field.enableVoiceModeAria': 'Activer le mode vocal',
|
|
||||||
'settings.voice.page.field.enableVoiceMode': 'Activer le mode vocal',
|
|
||||||
'settings.voice.page.field.provider': 'Fournisseur',
|
'settings.voice.page.field.provider': 'Fournisseur',
|
||||||
'settings.voice.page.provider.browser': 'Navigateur',
|
'settings.voice.page.provider.browser': 'Navigateur',
|
||||||
'settings.voice.page.provider.custom': 'Personnalisé',
|
'settings.voice.page.provider.custom': 'Personnalisé',
|
||||||
'settings.voice.page.provider.say': 'Dire',
|
'settings.voice.page.provider.say': 'Dire',
|
||||||
'settings.voice.page.provider.server': 'Serveur',
|
'settings.voice.page.provider.server': 'Serveur',
|
||||||
|
'settings.voice.page.provider.local': 'Local',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': 'Transcription locale sur le serveur OpenChamber. Les modèles se téléchargent automatiquement ; aucune clé d\'API requise.',
|
||||||
|
'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro, anglais). Le modèle se télécharge automatiquement ; aucune clé d’API requise.',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (anglais)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 langues européennes)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingue)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (multilingue)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': 'Idéal pour l’anglais',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': 'Idéal multilingue',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': 'Précision',
|
||||||
|
'settings.voice.page.stt.meta.speed': 'Vitesse',
|
||||||
|
'settings.voice.page.stt.modelInstalled': 'Modèle installé',
|
||||||
|
'settings.voice.page.stt.modelDownloading': 'Téléchargement du modèle...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': 'Téléchargement du modèle... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': 'Modèle non téléchargé',
|
||||||
|
'settings.voice.page.stt.modelDownload': 'Télécharger',
|
||||||
|
'settings.voice.page.stt.modelDelete': 'Supprimer le modèle',
|
||||||
|
'settings.voice.page.stt.modelRetry': 'Réessayer',
|
||||||
'settings.voice.page.provider.wasm': 'Local',
|
'settings.voice.page.provider.wasm': 'Local',
|
||||||
'settings.voice.page.stt.wasmModel': 'Modèle Whisper',
|
'settings.voice.page.stt.wasmModel': 'Modèle Whisper',
|
||||||
'settings.voice.page.stt.wasmLoaded': 'Modèle chargé, prêt',
|
'settings.voice.page.stt.wasmLoaded': 'Modèle chargé, prêt',
|
||||||
|
|||||||
@@ -1669,6 +1669,18 @@ export const dict = {
|
|||||||
'chat.chatInput.placeholder.chat': '@ pour les fichiers/agents ; / pour les commandes et les skills ; ! pour shell ; # pour les extraits',
|
'chat.chatInput.placeholder.chat': '@ pour les fichiers/agents ; / pour les commandes et les skills ; ! pour shell ; # pour les extraits',
|
||||||
'chat.chatInput.placeholder.chatCompact': 'Utiliser @ / ! # pour les aides',
|
'chat.chatInput.placeholder.chatCompact': 'Utiliser @ / ! # pour les aides',
|
||||||
'chat.chatInput.placeholder.selectSession': 'Sélectionnez ou créez une session pour commencer à discuter',
|
'chat.chatInput.placeholder.selectSession': 'Sélectionnez ou créez une session pour commencer à discuter',
|
||||||
|
'chat.dictation.start': 'Démarrer la dictée',
|
||||||
|
'chat.dictation.overlayAria': 'Dictée',
|
||||||
|
'chat.dictation.downloadingModel': 'Téléchargement du modèle vocal...',
|
||||||
|
'chat.dictation.downloadingModelProgress': 'Téléchargement du modèle vocal... {percent}%',
|
||||||
|
'chat.dictation.listening': 'Écoute...',
|
||||||
|
'chat.dictation.processing': 'Transcription...',
|
||||||
|
'chat.dictation.failed': 'Échec de la transcription',
|
||||||
|
'chat.dictation.cancel': 'Abandonner la dictée',
|
||||||
|
'chat.dictation.insert': 'Insérer la transcription',
|
||||||
|
'chat.dictation.insertAndSend': 'Insérer et envoyer',
|
||||||
|
'chat.dictation.retry': 'Réessayer la transcription',
|
||||||
|
'chat.dictation.discard': 'Abandonner l\'enregistrement',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ Ajouter un nouvel extrait',
|
'chat.snippetAutocomplete.action.addNew': '+ Ajouter un nouvel extrait',
|
||||||
'chat.snippetAutocomplete.empty': 'Aucun extrait trouvé',
|
'chat.snippetAutocomplete.empty': 'Aucun extrait trouvé',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ naviguer • Entrer sélectionner • Esc fermer',
|
'chat.snippetAutocomplete.footer': '↑↓ naviguer • Entrer sélectionner • Esc fermer',
|
||||||
|
|||||||
@@ -1518,16 +1518,33 @@ export const settingsDict = {
|
|||||||
'settings.notifications.page.toast.backgroundDisabled': 'バックグラウンド通知を無効化しました',
|
'settings.notifications.page.toast.backgroundDisabled': 'バックグラウンド通知を無効化しました',
|
||||||
'settings.notifications.page.testNotification.title': 'テスト通知',
|
'settings.notifications.page.testNotification.title': 'テスト通知',
|
||||||
'settings.notifications.page.testNotification.body': 'これは OpenChamber からのテスト通知です。',
|
'settings.notifications.page.testNotification.body': 'これは OpenChamber からのテスト通知です。',
|
||||||
'settings.voice.page.section.voiceSetup': '音声設定',
|
|
||||||
'settings.voice.page.section.speechRecognition': '音声認識',
|
'settings.voice.page.section.speechRecognition': '音声認識',
|
||||||
|
'settings.voice.page.field.enableVoiceInput': '音声入力を有効にする',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': '音声入力(ディクテーション)を有効にする',
|
||||||
'settings.voice.page.section.playbackAndSummary': '再生',
|
'settings.voice.page.section.playbackAndSummary': '再生',
|
||||||
'settings.voice.page.field.enableVoiceModeAria': '音声モードを有効化',
|
|
||||||
'settings.voice.page.field.enableVoiceMode': '音声モードを有効化',
|
|
||||||
'settings.voice.page.field.provider': 'プロバイダー',
|
'settings.voice.page.field.provider': 'プロバイダー',
|
||||||
'settings.voice.page.provider.browser': 'ブラウザ',
|
'settings.voice.page.provider.browser': 'ブラウザ',
|
||||||
'settings.voice.page.provider.custom': 'カスタム',
|
'settings.voice.page.provider.custom': 'カスタム',
|
||||||
'settings.voice.page.provider.say': 'Say',
|
'settings.voice.page.provider.say': 'Say',
|
||||||
'settings.voice.page.provider.server': 'サーバー',
|
'settings.voice.page.provider.server': 'サーバー',
|
||||||
|
'settings.voice.page.provider.local': 'ローカル',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': 'OpenChamber サーバー上でローカルに文字起こしします。モデルは自動でダウンロードされ、API キーは不要です。',
|
||||||
|
'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します(Kokoro、英語)。モデルは自動でダウンロードされ、API キーは不要です。',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英語)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(ヨーロッパ25言語)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多言語)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多言語)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': '英語に最適',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': '多言語に最適',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': '精度',
|
||||||
|
'settings.voice.page.stt.meta.speed': '速度',
|
||||||
|
'settings.voice.page.stt.modelInstalled': 'モデルはインストール済み',
|
||||||
|
'settings.voice.page.stt.modelDownloading': 'モデルをダウンロード中...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': 'モデルをダウンロード中... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': 'モデル未ダウンロード',
|
||||||
|
'settings.voice.page.stt.modelDownload': 'ダウンロード',
|
||||||
|
'settings.voice.page.stt.modelDelete': 'モデルを削除',
|
||||||
|
'settings.voice.page.stt.modelRetry': '再試行',
|
||||||
'settings.voice.page.provider.wasm': 'ローカル',
|
'settings.voice.page.provider.wasm': 'ローカル',
|
||||||
'settings.voice.page.stt.wasmModel': 'Whisper モデル',
|
'settings.voice.page.stt.wasmModel': 'Whisper モデル',
|
||||||
'settings.voice.page.stt.wasmLoaded': 'モデル読み込み完了',
|
'settings.voice.page.stt.wasmLoaded': 'モデル読み込み完了',
|
||||||
|
|||||||
@@ -1869,6 +1869,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'chat.chatInput.placeholder.chat': '@でファイル/エージェント、/でコマンド/スキル、!でシェル、#でスニペット',
|
'chat.chatInput.placeholder.chat': '@でファイル/エージェント、/でコマンド/スキル、!でシェル、#でスニペット',
|
||||||
'chat.chatInput.placeholder.chatCompact': '@ / ! # でヘルパーを使用',
|
'chat.chatInput.placeholder.chatCompact': '@ / ! # でヘルパーを使用',
|
||||||
'chat.chatInput.placeholder.selectSession': 'セッションを選択または作成してチャットを開始',
|
'chat.chatInput.placeholder.selectSession': 'セッションを選択または作成してチャットを開始',
|
||||||
|
'chat.dictation.start': '音声入力を開始',
|
||||||
|
'chat.dictation.overlayAria': '音声入力',
|
||||||
|
'chat.dictation.downloadingModel': '音声モデルをダウンロード中...',
|
||||||
|
'chat.dictation.downloadingModelProgress': '音声モデルをダウンロード中... {percent}%',
|
||||||
|
'chat.dictation.listening': '聞き取り中...',
|
||||||
|
'chat.dictation.processing': '文字起こし中...',
|
||||||
|
'chat.dictation.failed': '文字起こしに失敗しました',
|
||||||
|
'chat.dictation.cancel': '音声入力を破棄',
|
||||||
|
'chat.dictation.insert': 'テキストを挿入',
|
||||||
|
'chat.dictation.insertAndSend': '挿入して送信',
|
||||||
|
'chat.dictation.retry': '文字起こしを再試行',
|
||||||
|
'chat.dictation.discard': '録音を破棄',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ 新しいスニペットを追加',
|
'chat.snippetAutocomplete.action.addNew': '+ 新しいスニペットを追加',
|
||||||
'chat.snippetAutocomplete.empty': 'スニペットが見つかりません',
|
'chat.snippetAutocomplete.empty': 'スニペットが見つかりません',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ 移動 • Enter 選択 • Esc 閉じる',
|
'chat.snippetAutocomplete.footer': '↑↓ 移動 • Enter 選択 • Esc 閉じる',
|
||||||
|
|||||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
|||||||
'settings.notifications.page.toast.backgroundDisabled': '백그라운드 알림이 비활성화되었습니다',
|
'settings.notifications.page.toast.backgroundDisabled': '백그라운드 알림이 비활성화되었습니다',
|
||||||
'settings.notifications.page.testNotification.title': '테스트 알림',
|
'settings.notifications.page.testNotification.title': '테스트 알림',
|
||||||
'settings.notifications.page.testNotification.body': 'OpenChamber의 테스트 알림입니다.',
|
'settings.notifications.page.testNotification.body': 'OpenChamber의 테스트 알림입니다.',
|
||||||
'settings.voice.page.section.voiceSetup': '음성',
|
|
||||||
'settings.voice.page.section.speechRecognition': '음성 인식',
|
'settings.voice.page.section.speechRecognition': '음성 인식',
|
||||||
|
'settings.voice.page.field.enableVoiceInput': '음성 입력 사용',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': '음성 입력(받아쓰기) 사용',
|
||||||
'settings.voice.page.section.playbackAndSummary': '재생',
|
'settings.voice.page.section.playbackAndSummary': '재생',
|
||||||
'settings.voice.page.field.enableVoiceModeAria': '음성 모드 활성화',
|
|
||||||
'settings.voice.page.field.enableVoiceMode': '음성 모드 활성화',
|
|
||||||
'settings.voice.page.field.provider': '프로바이더',
|
'settings.voice.page.field.provider': '프로바이더',
|
||||||
'settings.voice.page.provider.browser': '브라우저',
|
'settings.voice.page.provider.browser': '브라우저',
|
||||||
'settings.voice.page.provider.custom': '사용자 정의',
|
'settings.voice.page.provider.custom': '사용자 정의',
|
||||||
'settings.voice.page.provider.say': 'Say',
|
'settings.voice.page.provider.say': 'Say',
|
||||||
'settings.voice.page.provider.server': '서버',
|
'settings.voice.page.provider.server': '서버',
|
||||||
|
'settings.voice.page.provider.local': '로컬',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': 'OpenChamber 서버에서 로컬로 변환합니다. 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.',
|
||||||
|
'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(Kokoro, 영어). 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (영어)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (유럽 25개 언어)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base (다국어)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (다국어)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': '영어에 최적',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': '다국어에 최적',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': '정확도',
|
||||||
|
'settings.voice.page.stt.meta.speed': '속도',
|
||||||
|
'settings.voice.page.stt.modelInstalled': '모델 설치됨',
|
||||||
|
'settings.voice.page.stt.modelDownloading': '모델 다운로드 중...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': '모델 다운로드 중... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': '모델이 다운로드되지 않음',
|
||||||
|
'settings.voice.page.stt.modelDownload': '다운로드',
|
||||||
|
'settings.voice.page.stt.modelDelete': '모델 삭제',
|
||||||
|
'settings.voice.page.stt.modelRetry': '다시 시도',
|
||||||
'settings.voice.page.provider.wasm': '로컬',
|
'settings.voice.page.provider.wasm': '로컬',
|
||||||
'settings.voice.page.stt.wasmModel': 'Whisper 모델',
|
'settings.voice.page.stt.wasmModel': 'Whisper 모델',
|
||||||
'settings.voice.page.stt.wasmLoaded': '모델 로드됨',
|
'settings.voice.page.stt.wasmLoaded': '모델 로드됨',
|
||||||
|
|||||||
@@ -1873,6 +1873,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'chat.chatInput.placeholder.chat': '@ 파일/에이전트; / 명령 및 스킬; ! shell; # 스니펫',
|
'chat.chatInput.placeholder.chat': '@ 파일/에이전트; / 명령 및 스킬; ! shell; # 스니펫',
|
||||||
'chat.chatInput.placeholder.chatCompact': '@ / ! # 도우미 사용',
|
'chat.chatInput.placeholder.chatCompact': '@ / ! # 도우미 사용',
|
||||||
'chat.chatInput.placeholder.selectSession': '채팅을 시작할 세션을 선택하거나 새로 만드세요',
|
'chat.chatInput.placeholder.selectSession': '채팅을 시작할 세션을 선택하거나 새로 만드세요',
|
||||||
|
'chat.dictation.start': '받아쓰기 시작',
|
||||||
|
'chat.dictation.overlayAria': '받아쓰기',
|
||||||
|
'chat.dictation.downloadingModel': '음성 모델 다운로드 중...',
|
||||||
|
'chat.dictation.downloadingModelProgress': '음성 모델 다운로드 중... {percent}%',
|
||||||
|
'chat.dictation.listening': '듣는 중...',
|
||||||
|
'chat.dictation.processing': '변환 중...',
|
||||||
|
'chat.dictation.failed': '변환에 실패했습니다',
|
||||||
|
'chat.dictation.cancel': '받아쓰기 취소',
|
||||||
|
'chat.dictation.insert': '텍스트 삽입',
|
||||||
|
'chat.dictation.insertAndSend': '삽입 후 전송',
|
||||||
|
'chat.dictation.retry': '변환 다시 시도',
|
||||||
|
'chat.dictation.discard': '녹음 삭제',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ 새 스니펫 추가',
|
'chat.snippetAutocomplete.action.addNew': '+ 새 스니펫 추가',
|
||||||
'chat.snippetAutocomplete.empty': '스니펫을 찾을 수 없음',
|
'chat.snippetAutocomplete.empty': '스니펫을 찾을 수 없음',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ 이동 • Enter 선택 • Esc 닫기',
|
'chat.snippetAutocomplete.footer': '↑↓ 이동 • Enter 선택 • Esc 닫기',
|
||||||
|
|||||||
@@ -1743,8 +1743,6 @@ export const settingsDict = {
|
|||||||
'settings.voice.page.field.apiKeyHintUsingConfig': 'Użyto klucza z konfiguracji',
|
'settings.voice.page.field.apiKeyHintUsingConfig': 'Użyto klucza z konfiguracji',
|
||||||
'settings.voice.page.field.auto': 'Auto',
|
'settings.voice.page.field.auto': 'Auto',
|
||||||
'settings.voice.page.field.configuredAbove': 'Skonfigurowano powyżej',
|
'settings.voice.page.field.configuredAbove': 'Skonfigurowano powyżej',
|
||||||
'settings.voice.page.field.enableVoiceMode': 'Włącz tryb głosowy',
|
|
||||||
'settings.voice.page.field.enableVoiceModeAria': 'Włącz tryb głosowy',
|
|
||||||
'settings.voice.page.field.language': 'Język',
|
'settings.voice.page.field.language': 'Język',
|
||||||
'settings.voice.page.field.messageReadAloudButton': 'Przycisk czytania wiadomości na głos',
|
'settings.voice.page.field.messageReadAloudButton': 'Przycisk czytania wiadomości na głos',
|
||||||
'settings.voice.page.field.messageReadAloudButtonAria': 'Przycisk czytania wiadomości na głos',
|
'settings.voice.page.field.messageReadAloudButtonAria': 'Przycisk czytania wiadomości na głos',
|
||||||
@@ -1782,6 +1780,24 @@ export const settingsDict = {
|
|||||||
'settings.voice.page.provider.custom': 'Własny',
|
'settings.voice.page.provider.custom': 'Własny',
|
||||||
'settings.voice.page.provider.say': 'Say',
|
'settings.voice.page.provider.say': 'Say',
|
||||||
'settings.voice.page.provider.server': 'Serwer',
|
'settings.voice.page.provider.server': 'Serwer',
|
||||||
|
'settings.voice.page.provider.local': 'Lokalny',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': 'Transkrypcja lokalna na serwerze OpenChamber. Modele pobierają się automatycznie; klucz API nie jest potrzebny.',
|
||||||
|
'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro, angielski). Model pobiera się automatycznie; klucz API nie jest potrzebny.',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (angielski)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 języków europejskich)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base (wielojęzyczny)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny (wielojęzyczny)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': 'Najlepszy do angielskiego',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': 'Najlepszy wielojęzyczny',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': 'Dokładność',
|
||||||
|
'settings.voice.page.stt.meta.speed': 'Szybkość',
|
||||||
|
'settings.voice.page.stt.modelInstalled': 'Model zainstalowany',
|
||||||
|
'settings.voice.page.stt.modelDownloading': 'Pobieranie modelu...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': 'Pobieranie modelu... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': 'Model niepobrany',
|
||||||
|
'settings.voice.page.stt.modelDownload': 'Pobierz',
|
||||||
|
'settings.voice.page.stt.modelDelete': 'Usuń model',
|
||||||
|
'settings.voice.page.stt.modelRetry': 'Ponów',
|
||||||
'settings.voice.page.provider.wasm': 'Lokalny',
|
'settings.voice.page.provider.wasm': 'Lokalny',
|
||||||
'settings.voice.page.stt.wasmModel': 'Model Whisper',
|
'settings.voice.page.stt.wasmModel': 'Model Whisper',
|
||||||
'settings.voice.page.stt.wasmLoaded': 'Model załadowany',
|
'settings.voice.page.stt.wasmLoaded': 'Model załadowany',
|
||||||
@@ -1792,7 +1808,8 @@ export const settingsDict = {
|
|||||||
'settings.voice.page.stt.wasmRetry': 'Spróbuj ponownie',
|
'settings.voice.page.stt.wasmRetry': 'Spróbuj ponownie',
|
||||||
'settings.voice.page.section.playbackAndSummary': 'Odtwarzanie',
|
'settings.voice.page.section.playbackAndSummary': 'Odtwarzanie',
|
||||||
'settings.voice.page.section.speechRecognition': 'Rozpoznawanie mowy',
|
'settings.voice.page.section.speechRecognition': 'Rozpoznawanie mowy',
|
||||||
'settings.voice.page.section.voiceSetup': 'Konfiguracja głosu',
|
'settings.voice.page.field.enableVoiceInput': 'Włącz wprowadzanie głosowe',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': 'Włącz wprowadzanie głosowe (dyktowanie)',
|
||||||
'settings.voice.page.tooltip.browser': 'Darmowe, offline, ograniczone wsparcie mobilne.',
|
'settings.voice.page.tooltip.browser': 'Darmowe, offline, ograniczone wsparcie mobilne.',
|
||||||
'settings.voice.page.tooltip.custom': 'Serwer zgodny z OpenAI (na przykład Kokoro).',
|
'settings.voice.page.tooltip.custom': 'Serwer zgodny z OpenAI (na przykład Kokoro).',
|
||||||
'settings.voice.page.tooltip.openai': 'Wysoka jakość, gotowe na urządzenia mobilne, wymaga klucza API.',
|
'settings.voice.page.tooltip.openai': 'Wysoka jakość, gotowe na urządzenia mobilne, wymaga klucza API.',
|
||||||
|
|||||||
@@ -1091,6 +1091,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'chat.chatInput.placeholder.chatCompact': 'Użyj @ / ! # dla pomocników',
|
'chat.chatInput.placeholder.chatCompact': 'Użyj @ / ! # dla pomocników',
|
||||||
'chat.chatInput.placeholder.selectSession': 'Wybierz lub utwórz sesję, aby zacząć czatować',
|
'chat.chatInput.placeholder.selectSession': 'Wybierz lub utwórz sesję, aby zacząć czatować',
|
||||||
'chat.chatInput.placeholder.shell': 'Wpisz polecenie powłoki...',
|
'chat.chatInput.placeholder.shell': 'Wpisz polecenie powłoki...',
|
||||||
|
'chat.dictation.start': 'Rozpocznij dyktowanie',
|
||||||
|
'chat.dictation.overlayAria': 'Dyktowanie',
|
||||||
|
'chat.dictation.downloadingModel': 'Pobieranie modelu mowy...',
|
||||||
|
'chat.dictation.downloadingModelProgress': 'Pobieranie modelu mowy... {percent}%',
|
||||||
|
'chat.dictation.listening': 'Słucham...',
|
||||||
|
'chat.dictation.processing': 'Transkrybowanie...',
|
||||||
|
'chat.dictation.failed': 'Transkrypcja nie powiodła się',
|
||||||
|
'chat.dictation.cancel': 'Odrzuć dyktowanie',
|
||||||
|
'chat.dictation.insert': 'Wstaw transkrypcję',
|
||||||
|
'chat.dictation.insertAndSend': 'Wstaw i wyślij',
|
||||||
|
'chat.dictation.retry': 'Ponów transkrypcję',
|
||||||
|
'chat.dictation.discard': 'Odrzuć nagranie',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ Dodaj nowy fragment',
|
'chat.snippetAutocomplete.action.addNew': '+ Dodaj nowy fragment',
|
||||||
'chat.snippetAutocomplete.empty': 'Nie znaleziono fragmentów',
|
'chat.snippetAutocomplete.empty': 'Nie znaleziono fragmentów',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ nawigacja • Enter wybierz • Esc zamknij',
|
'chat.snippetAutocomplete.footer': '↑↓ nawigacja • Enter wybierz • Esc zamknij',
|
||||||
|
|||||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
|||||||
"settings.notifications.page.toast.backgroundDisabled": "Notificações em segundo plano desabilitadas",
|
"settings.notifications.page.toast.backgroundDisabled": "Notificações em segundo plano desabilitadas",
|
||||||
"settings.notifications.page.testNotification.title": "Notificação de teste",
|
"settings.notifications.page.testNotification.title": "Notificação de teste",
|
||||||
"settings.notifications.page.testNotification.body": "Esta é uma notificação de teste do OpenChamber.",
|
"settings.notifications.page.testNotification.body": "Esta é uma notificação de teste do OpenChamber.",
|
||||||
"settings.voice.page.section.voiceSetup": "Configurações de voz",
|
|
||||||
"settings.voice.page.section.speechRecognition": "Reconhecimento de voz",
|
"settings.voice.page.section.speechRecognition": "Reconhecimento de voz",
|
||||||
|
"settings.voice.page.field.enableVoiceInput": "Ativar entrada por voz",
|
||||||
|
"settings.voice.page.field.enableVoiceInputAria": "Ativar entrada por voz (ditado)",
|
||||||
"settings.voice.page.section.playbackAndSummary": "Reprodução",
|
"settings.voice.page.section.playbackAndSummary": "Reprodução",
|
||||||
"settings.voice.page.field.enableVoiceModeAria": "Ativar modo de voz",
|
|
||||||
"settings.voice.page.field.enableVoiceMode": "Ativar modo de voz",
|
|
||||||
"settings.voice.page.field.provider": "Provedor",
|
"settings.voice.page.field.provider": "Provedor",
|
||||||
"settings.voice.page.provider.browser": "Navegador",
|
"settings.voice.page.provider.browser": "Navegador",
|
||||||
"settings.voice.page.provider.custom": "Personalizado",
|
"settings.voice.page.provider.custom": "Personalizado",
|
||||||
"settings.voice.page.provider.say": "Falar",
|
"settings.voice.page.provider.say": "Falar",
|
||||||
"settings.voice.page.provider.server": "Servidor",
|
"settings.voice.page.provider.server": "Servidor",
|
||||||
|
"settings.voice.page.provider.local": "Local",
|
||||||
|
"settings.voice.page.tooltip.sttLocal": "Transcrição local no servidor do OpenChamber. Os modelos são baixados automaticamente; não é necessária chave de API.",
|
||||||
|
"settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro, inglês). O modelo é baixado automaticamente; não é necessária chave de API.",
|
||||||
|
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglês)",
|
||||||
|
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeus)",
|
||||||
|
"settings.voice.page.stt.model.whisperBase": "Whisper base (multilíngue)",
|
||||||
|
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (multilíngue)",
|
||||||
|
"settings.voice.page.stt.badge.bestForEnglish": "Melhor para inglês",
|
||||||
|
"settings.voice.page.stt.badge.bestForMultilingual": "Melhor multilíngue",
|
||||||
|
"settings.voice.page.stt.meta.accuracy": "Precisão",
|
||||||
|
"settings.voice.page.stt.meta.speed": "Velocidade",
|
||||||
|
"settings.voice.page.stt.modelInstalled": "Modelo instalado",
|
||||||
|
"settings.voice.page.stt.modelDownloading": "Baixando modelo...",
|
||||||
|
"settings.voice.page.stt.modelDownloadingProgress": "Baixando modelo... {percent}%",
|
||||||
|
"settings.voice.page.stt.modelNotInstalled": "Modelo não baixado",
|
||||||
|
"settings.voice.page.stt.modelDownload": "Baixar",
|
||||||
|
"settings.voice.page.stt.modelDelete": "Excluir modelo",
|
||||||
|
"settings.voice.page.stt.modelRetry": "Tentar novamente",
|
||||||
"settings.voice.page.provider.wasm": "Local",
|
"settings.voice.page.provider.wasm": "Local",
|
||||||
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
|
"settings.voice.page.stt.wasmModel": "Modelo Whisper",
|
||||||
"settings.voice.page.stt.wasmLoaded": "Modelo carregado",
|
"settings.voice.page.stt.wasmLoaded": "Modelo carregado",
|
||||||
|
|||||||
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'diffView.reviewDialog.actions.starting': 'Iniciando...',
|
'diffView.reviewDialog.actions.starting': 'Iniciando...',
|
||||||
'diffView.reviewDialog.toast.noSessionDirectory': 'O diretório da sessão está indisponível',
|
'diffView.reviewDialog.toast.noSessionDirectory': 'O diretório da sessão está indisponível',
|
||||||
'diffView.reviewDialog.toast.startFailed': 'Falha ao iniciar o fluxo de revisão',
|
'diffView.reviewDialog.toast.startFailed': 'Falha ao iniciar o fluxo de revisão',
|
||||||
|
'chat.dictation.start': 'Iniciar ditado',
|
||||||
|
'chat.dictation.overlayAria': 'Ditado',
|
||||||
|
'chat.dictation.downloadingModel': 'Baixando o modelo de fala...',
|
||||||
|
'chat.dictation.downloadingModelProgress': 'Baixando o modelo de fala... {percent}%',
|
||||||
|
'chat.dictation.listening': 'Ouvindo...',
|
||||||
|
'chat.dictation.processing': 'Transcrevendo...',
|
||||||
|
'chat.dictation.failed': 'Falha na transcrição',
|
||||||
|
'chat.dictation.cancel': 'Descartar ditado',
|
||||||
|
'chat.dictation.insert': 'Inserir transcrição',
|
||||||
|
'chat.dictation.insertAndSend': 'Inserir e enviar',
|
||||||
|
'chat.dictation.retry': 'Tentar transcrever novamente',
|
||||||
|
'chat.dictation.discard': 'Descartar gravação',
|
||||||
'chat.history.loadOlder': 'Carregar mensagens anteriores',
|
'chat.history.loadOlder': 'Carregar mensagens anteriores',
|
||||||
'chat.autoReview.title': 'O ciclo de revisão de código está em andamento',
|
'chat.autoReview.title': 'O ciclo de revisão de código está em andamento',
|
||||||
'chat.autoReview.status.waitingForReviewer': 'Aguardando o revisor',
|
'chat.autoReview.status.waitingForReviewer': 'Aguardando o revisor',
|
||||||
|
|||||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
|||||||
"settings.notifications.page.toast.backgroundDisabled": "Фонові сповіщення вимкнено",
|
"settings.notifications.page.toast.backgroundDisabled": "Фонові сповіщення вимкнено",
|
||||||
"settings.notifications.page.testNotification.title": "Тестове сповіщення",
|
"settings.notifications.page.testNotification.title": "Тестове сповіщення",
|
||||||
"settings.notifications.page.testNotification.body": "Це тестове сповіщення від OpenChamber.",
|
"settings.notifications.page.testNotification.body": "Це тестове сповіщення від OpenChamber.",
|
||||||
"settings.voice.page.section.voiceSetup": "Налаштування голосу",
|
|
||||||
"settings.voice.page.section.speechRecognition": "Розпізнавання мовлення",
|
"settings.voice.page.section.speechRecognition": "Розпізнавання мовлення",
|
||||||
|
"settings.voice.page.field.enableVoiceInput": "Увімкнути голосовий ввід",
|
||||||
|
"settings.voice.page.field.enableVoiceInputAria": "Увімкнути голосовий ввід (диктування)",
|
||||||
"settings.voice.page.section.playbackAndSummary": "Відтворення",
|
"settings.voice.page.section.playbackAndSummary": "Відтворення",
|
||||||
"settings.voice.page.field.enableVoiceModeAria": "Увімкнути голосовий режим",
|
|
||||||
"settings.voice.page.field.enableVoiceMode": "Увімкнути голосовий режим",
|
|
||||||
"settings.voice.page.field.provider": "Провайдер",
|
"settings.voice.page.field.provider": "Провайдер",
|
||||||
"settings.voice.page.provider.browser": "Браузер",
|
"settings.voice.page.provider.browser": "Браузер",
|
||||||
"settings.voice.page.provider.custom": "Власний",
|
"settings.voice.page.provider.custom": "Власний",
|
||||||
"settings.voice.page.provider.say": "Say",
|
"settings.voice.page.provider.say": "Say",
|
||||||
"settings.voice.page.provider.server": "Сервер",
|
"settings.voice.page.provider.server": "Сервер",
|
||||||
|
"settings.voice.page.provider.local": "Локальний",
|
||||||
|
"settings.voice.page.tooltip.sttLocal": "Локальна розшифровка на сервері OpenChamber. Моделі завантажуються автоматично; ключ API не потрібен.",
|
||||||
|
"settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro, англійська). Модель завантажується автоматично; ключ API не потрібен.",
|
||||||
|
"settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (англійська)",
|
||||||
|
"settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 європейських мов)",
|
||||||
|
"settings.voice.page.stt.model.whisperBase": "Whisper base (мультимовна)",
|
||||||
|
"settings.voice.page.stt.model.whisperTiny": "Whisper tiny (мультимовна)",
|
||||||
|
"settings.voice.page.stt.badge.bestForEnglish": "Найкраща для англійської",
|
||||||
|
"settings.voice.page.stt.badge.bestForMultilingual": "Найкраща мультимовна",
|
||||||
|
"settings.voice.page.stt.meta.accuracy": "Точність",
|
||||||
|
"settings.voice.page.stt.meta.speed": "Швидкість",
|
||||||
|
"settings.voice.page.stt.modelInstalled": "Модель встановлено",
|
||||||
|
"settings.voice.page.stt.modelDownloading": "Завантаження моделі...",
|
||||||
|
"settings.voice.page.stt.modelDownloadingProgress": "Завантаження моделі... {percent}%",
|
||||||
|
"settings.voice.page.stt.modelNotInstalled": "Модель не завантажено",
|
||||||
|
"settings.voice.page.stt.modelDownload": "Завантажити",
|
||||||
|
"settings.voice.page.stt.modelDelete": "Видалити модель",
|
||||||
|
"settings.voice.page.stt.modelRetry": "Повторити",
|
||||||
"settings.voice.page.provider.wasm": "Локально",
|
"settings.voice.page.provider.wasm": "Локально",
|
||||||
"settings.voice.page.stt.wasmModel": "Модель Whisper",
|
"settings.voice.page.stt.wasmModel": "Модель Whisper",
|
||||||
"settings.voice.page.stt.wasmLoaded": "Модель завантажено",
|
"settings.voice.page.stt.wasmLoaded": "Модель завантажено",
|
||||||
|
|||||||
@@ -1245,6 +1245,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'diffView.reviewDialog.actions.starting': 'Запуск...',
|
'diffView.reviewDialog.actions.starting': 'Запуск...',
|
||||||
'diffView.reviewDialog.toast.noSessionDirectory': 'Директорія сесії недоступна',
|
'diffView.reviewDialog.toast.noSessionDirectory': 'Директорія сесії недоступна',
|
||||||
'diffView.reviewDialog.toast.startFailed': 'Не вдалося запустити review flow',
|
'diffView.reviewDialog.toast.startFailed': 'Не вдалося запустити review flow',
|
||||||
|
'chat.dictation.start': 'Почати диктування',
|
||||||
|
'chat.dictation.overlayAria': 'Диктування',
|
||||||
|
'chat.dictation.downloadingModel': 'Завантаження мовної моделі...',
|
||||||
|
'chat.dictation.downloadingModelProgress': 'Завантаження мовної моделі... {percent}%',
|
||||||
|
'chat.dictation.listening': 'Слухаю...',
|
||||||
|
'chat.dictation.processing': 'Розшифровка...',
|
||||||
|
'chat.dictation.failed': 'Не вдалося розшифрувати запис',
|
||||||
|
'chat.dictation.cancel': 'Відхилити диктування',
|
||||||
|
'chat.dictation.insert': 'Вставити текст',
|
||||||
|
'chat.dictation.insertAndSend': 'Вставити й надіслати',
|
||||||
|
'chat.dictation.retry': 'Повторити розшифровку',
|
||||||
|
'chat.dictation.discard': 'Відхилити запис',
|
||||||
'chat.history.loadOlder': 'Завантажити ще',
|
'chat.history.loadOlder': 'Завантажити ще',
|
||||||
'chat.autoReview.title': 'Цикл код-ревʼю триває',
|
'chat.autoReview.title': 'Цикл код-ревʼю триває',
|
||||||
'chat.autoReview.status.waitingForReviewer': 'Очікуємо ревʼювера',
|
'chat.autoReview.status.waitingForReviewer': 'Очікуємо ревʼювера',
|
||||||
|
|||||||
@@ -1485,16 +1485,33 @@ export const settingsDict = {
|
|||||||
'settings.notifications.page.toast.backgroundDisabled': '后台通知已关闭',
|
'settings.notifications.page.toast.backgroundDisabled': '后台通知已关闭',
|
||||||
'settings.notifications.page.testNotification.title': '测试通知',
|
'settings.notifications.page.testNotification.title': '测试通知',
|
||||||
'settings.notifications.page.testNotification.body': '这是来自 OpenChamber 的测试通知。',
|
'settings.notifications.page.testNotification.body': '这是来自 OpenChamber 的测试通知。',
|
||||||
'settings.voice.page.section.voiceSetup': '语音设置',
|
|
||||||
'settings.voice.page.section.speechRecognition': '语音识别',
|
'settings.voice.page.section.speechRecognition': '语音识别',
|
||||||
|
'settings.voice.page.field.enableVoiceInput': '启用语音输入',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': '启用语音输入(听写)',
|
||||||
'settings.voice.page.section.playbackAndSummary': '播放',
|
'settings.voice.page.section.playbackAndSummary': '播放',
|
||||||
'settings.voice.page.field.enableVoiceModeAria': '启用语音模式',
|
|
||||||
'settings.voice.page.field.enableVoiceMode': '启用语音模式',
|
|
||||||
'settings.voice.page.field.provider': '提供方',
|
'settings.voice.page.field.provider': '提供方',
|
||||||
'settings.voice.page.provider.browser': '浏览器',
|
'settings.voice.page.provider.browser': '浏览器',
|
||||||
'settings.voice.page.provider.custom': '自定义',
|
'settings.voice.page.provider.custom': '自定义',
|
||||||
'settings.voice.page.provider.say': 'Say',
|
'settings.voice.page.provider.say': 'Say',
|
||||||
'settings.voice.page.provider.server': '服务器',
|
'settings.voice.page.provider.server': '服务器',
|
||||||
|
'settings.voice.page.provider.local': '本地',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 服务器上本地转写。模型自动下载,无需 API 密钥。',
|
||||||
|
'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音(Kokoro,英语)。模型自动下载,无需 API 密钥。',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英语)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(25 种欧洲语言)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多语言)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多语言)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': '英语最佳',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': '多语言最佳',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': '准确度',
|
||||||
|
'settings.voice.page.stt.meta.speed': '速度',
|
||||||
|
'settings.voice.page.stt.modelInstalled': '模型已安装',
|
||||||
|
'settings.voice.page.stt.modelDownloading': '正在下载模型...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': '正在下载模型... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': '模型未下载',
|
||||||
|
'settings.voice.page.stt.modelDownload': '下载',
|
||||||
|
'settings.voice.page.stt.modelDelete': '删除模型',
|
||||||
|
'settings.voice.page.stt.modelRetry': '重试',
|
||||||
'settings.voice.page.provider.wasm': '本地',
|
'settings.voice.page.provider.wasm': '本地',
|
||||||
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
|
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
|
||||||
'settings.voice.page.stt.wasmLoaded': '模型已加载',
|
'settings.voice.page.stt.wasmLoaded': '模型已加载',
|
||||||
|
|||||||
@@ -1839,6 +1839,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'chat.chatInput.placeholder.chat': '@ 用于文件/智能体;/ 用于命令和技能;! 用于 shell;# 用于代码片段',
|
'chat.chatInput.placeholder.chat': '@ 用于文件/智能体;/ 用于命令和技能;! 用于 shell;# 用于代码片段',
|
||||||
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 辅助',
|
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 辅助',
|
||||||
'chat.chatInput.placeholder.selectSession': '选择或创建会话以开始聊天',
|
'chat.chatInput.placeholder.selectSession': '选择或创建会话以开始聊天',
|
||||||
|
'chat.dictation.start': '开始语音输入',
|
||||||
|
'chat.dictation.overlayAria': '语音输入',
|
||||||
|
'chat.dictation.downloadingModel': '正在下载语音模型...',
|
||||||
|
'chat.dictation.downloadingModelProgress': '正在下载语音模型... {percent}%',
|
||||||
|
'chat.dictation.listening': '正在聆听...',
|
||||||
|
'chat.dictation.processing': '正在转写...',
|
||||||
|
'chat.dictation.failed': '转写失败',
|
||||||
|
'chat.dictation.cancel': '放弃语音输入',
|
||||||
|
'chat.dictation.insert': '插入文本',
|
||||||
|
'chat.dictation.insertAndSend': '插入并发送',
|
||||||
|
'chat.dictation.retry': '重试转写',
|
||||||
|
'chat.dictation.discard': '放弃录音',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ 新建代码片段',
|
'chat.snippetAutocomplete.action.addNew': '+ 新建代码片段',
|
||||||
'chat.snippetAutocomplete.empty': '未找到代码片段',
|
'chat.snippetAutocomplete.empty': '未找到代码片段',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ 导航 • Enter 选择 • Esc 关闭',
|
'chat.snippetAutocomplete.footer': '↑↓ 导航 • Enter 选择 • Esc 关闭',
|
||||||
|
|||||||
@@ -1401,16 +1401,33 @@
|
|||||||
'settings.notifications.page.toast.backgroundDisabled': '背景通知已關閉',
|
'settings.notifications.page.toast.backgroundDisabled': '背景通知已關閉',
|
||||||
'settings.notifications.page.testNotification.title': '測試通知',
|
'settings.notifications.page.testNotification.title': '測試通知',
|
||||||
'settings.notifications.page.testNotification.body': '這是來自 OpenChamber 的測試通知。',
|
'settings.notifications.page.testNotification.body': '這是來自 OpenChamber 的測試通知。',
|
||||||
'settings.voice.page.section.voiceSetup': '語音設定',
|
|
||||||
'settings.voice.page.section.speechRecognition': '語音辨識',
|
'settings.voice.page.section.speechRecognition': '語音辨識',
|
||||||
|
'settings.voice.page.field.enableVoiceInput': '啟用語音輸入',
|
||||||
|
'settings.voice.page.field.enableVoiceInputAria': '啟用語音輸入(聽寫)',
|
||||||
'settings.voice.page.section.playbackAndSummary': '播放',
|
'settings.voice.page.section.playbackAndSummary': '播放',
|
||||||
'settings.voice.page.field.enableVoiceModeAria': '啟用語音模式',
|
|
||||||
'settings.voice.page.field.enableVoiceMode': '啟用語音模式',
|
|
||||||
'settings.voice.page.field.provider': '提供方',
|
'settings.voice.page.field.provider': '提供方',
|
||||||
'settings.voice.page.provider.browser': '瀏覽器',
|
'settings.voice.page.provider.browser': '瀏覽器',
|
||||||
'settings.voice.page.provider.custom': '自訂',
|
'settings.voice.page.provider.custom': '自訂',
|
||||||
'settings.voice.page.provider.say': 'Say',
|
'settings.voice.page.provider.say': 'Say',
|
||||||
'settings.voice.page.provider.server': '伺服器',
|
'settings.voice.page.provider.server': '伺服器',
|
||||||
|
'settings.voice.page.provider.local': '本機',
|
||||||
|
'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 伺服器上本機轉寫。模型會自動下載,無需 API 金鑰。',
|
||||||
|
'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音(Kokoro,英文)。模型會自動下載,無需 API 金鑰。',
|
||||||
|
'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英文)',
|
||||||
|
'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(25 種歐洲語言)',
|
||||||
|
'settings.voice.page.stt.model.whisperBase': 'Whisper base(多語言)',
|
||||||
|
'settings.voice.page.stt.model.whisperTiny': 'Whisper tiny(多語言)',
|
||||||
|
'settings.voice.page.stt.badge.bestForEnglish': '英文最佳',
|
||||||
|
'settings.voice.page.stt.badge.bestForMultilingual': '多語言最佳',
|
||||||
|
'settings.voice.page.stt.meta.accuracy': '準確度',
|
||||||
|
'settings.voice.page.stt.meta.speed': '速度',
|
||||||
|
'settings.voice.page.stt.modelInstalled': '模型已安裝',
|
||||||
|
'settings.voice.page.stt.modelDownloading': '正在下載模型...',
|
||||||
|
'settings.voice.page.stt.modelDownloadingProgress': '正在下載模型... {percent}%',
|
||||||
|
'settings.voice.page.stt.modelNotInstalled': '模型未下載',
|
||||||
|
'settings.voice.page.stt.modelDownload': '下載',
|
||||||
|
'settings.voice.page.stt.modelDelete': '刪除模型',
|
||||||
|
'settings.voice.page.stt.modelRetry': '重試',
|
||||||
'settings.voice.page.provider.wasm': '本機',
|
'settings.voice.page.provider.wasm': '本機',
|
||||||
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
|
'settings.voice.page.stt.wasmModel': 'Whisper 模型',
|
||||||
'settings.voice.page.stt.wasmLoaded': '模型已載入',
|
'settings.voice.page.stt.wasmLoaded': '模型已載入',
|
||||||
|
|||||||
@@ -1843,6 +1843,18 @@ export const dict: Record<I18nKey, string> = {
|
|||||||
'chat.chatInput.placeholder.chat': '@ 用於檔案/Agent;/ 用於命令;! 用於 shell;# 用於程式片段',
|
'chat.chatInput.placeholder.chat': '@ 用於檔案/Agent;/ 用於命令;! 用於 shell;# 用於程式片段',
|
||||||
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 輔助',
|
'chat.chatInput.placeholder.chatCompact': '使用 @ / ! # 輔助',
|
||||||
'chat.chatInput.placeholder.selectSession': '選擇或建立會話以開始聊天',
|
'chat.chatInput.placeholder.selectSession': '選擇或建立會話以開始聊天',
|
||||||
|
'chat.dictation.start': '開始語音輸入',
|
||||||
|
'chat.dictation.overlayAria': '語音輸入',
|
||||||
|
'chat.dictation.downloadingModel': '正在下載語音模型...',
|
||||||
|
'chat.dictation.downloadingModelProgress': '正在下載語音模型... {percent}%',
|
||||||
|
'chat.dictation.listening': '正在聆聽...',
|
||||||
|
'chat.dictation.processing': '正在轉寫...',
|
||||||
|
'chat.dictation.failed': '轉寫失敗',
|
||||||
|
'chat.dictation.cancel': '放棄語音輸入',
|
||||||
|
'chat.dictation.insert': '插入文字',
|
||||||
|
'chat.dictation.insertAndSend': '插入並傳送',
|
||||||
|
'chat.dictation.retry': '重試轉寫',
|
||||||
|
'chat.dictation.discard': '放棄錄音',
|
||||||
'chat.snippetAutocomplete.action.addNew': '+ 新建程式片段',
|
'chat.snippetAutocomplete.action.addNew': '+ 新建程式片段',
|
||||||
'chat.snippetAutocomplete.empty': '未找到程式片段',
|
'chat.snippetAutocomplete.empty': '未找到程式片段',
|
||||||
'chat.snippetAutocomplete.footer': '↑↓ 導航 • Enter 選擇 • Esc 關閉',
|
'chat.snippetAutocomplete.footer': '↑↓ 導航 • Enter 選擇 • Esc 關閉',
|
||||||
|
|||||||
@@ -113,7 +113,10 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
|
|||||||
localStorage.removeItem('opencode-update-toast-dismissed-version');
|
localStorage.removeItem('opencode-update-toast-dismissed-version');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (settings.sttProvider === 'browser' || settings.sttProvider === 'server') {
|
if (typeof settings.dictationEnabled === 'boolean') {
|
||||||
|
localStorage.setItem('dictationEnabled', String(settings.dictationEnabled));
|
||||||
|
}
|
||||||
|
if (settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') {
|
||||||
localStorage.setItem('sttProvider', settings.sttProvider);
|
localStorage.setItem('sttProvider', settings.sttProvider);
|
||||||
}
|
}
|
||||||
if (typeof settings.sttServerUrl === 'string') {
|
if (typeof settings.sttServerUrl === 'string') {
|
||||||
@@ -122,18 +125,12 @@ const persistToLocalStorage = (settings: DesktopSettings) => {
|
|||||||
if (typeof settings.sttModel === 'string') {
|
if (typeof settings.sttModel === 'string') {
|
||||||
localStorage.setItem('sttModel', settings.sttModel);
|
localStorage.setItem('sttModel', settings.sttModel);
|
||||||
}
|
}
|
||||||
|
if (typeof settings.sttLocalModel === 'string') {
|
||||||
|
localStorage.setItem('sttLocalModel', settings.sttLocalModel);
|
||||||
|
}
|
||||||
if (typeof settings.sttLanguage === 'string') {
|
if (typeof settings.sttLanguage === 'string') {
|
||||||
localStorage.setItem('sttLanguage', settings.sttLanguage);
|
localStorage.setItem('sttLanguage', settings.sttLanguage);
|
||||||
}
|
}
|
||||||
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb)) {
|
|
||||||
localStorage.setItem('sttSilenceThresholdDb', String(settings.sttSilenceThresholdDb));
|
|
||||||
}
|
|
||||||
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs)) {
|
|
||||||
localStorage.setItem('sttSilenceHoldMs', String(settings.sttSilenceHoldMs));
|
|
||||||
}
|
|
||||||
if (typeof settings.sttTranscribeOnStop === 'boolean') {
|
|
||||||
localStorage.setItem('sttTranscribeOnStop', String(settings.sttTranscribeOnStop));
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const dispatchSettingsSynced = (settings: DesktopSettings): void => {
|
const dispatchSettingsSynced = (settings: DesktopSettings): void => {
|
||||||
@@ -614,7 +611,10 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
|
|||||||
}
|
}
|
||||||
if (configStoreApi && configStore) {
|
if (configStoreApi && configStore) {
|
||||||
const nextConfigState: Partial<typeof configStore> = {};
|
const nextConfigState: Partial<typeof configStore> = {};
|
||||||
if ((settings.sttProvider === 'browser' || settings.sttProvider === 'server') && settings.sttProvider !== configStore.sttProvider) {
|
if (typeof settings.dictationEnabled === 'boolean' && settings.dictationEnabled !== configStore.dictationEnabled) {
|
||||||
|
nextConfigState.dictationEnabled = settings.dictationEnabled;
|
||||||
|
}
|
||||||
|
if ((settings.sttProvider === 'local' || settings.sttProvider === 'openai-compatible') && settings.sttProvider !== configStore.sttProvider) {
|
||||||
nextConfigState.sttProvider = settings.sttProvider;
|
nextConfigState.sttProvider = settings.sttProvider;
|
||||||
}
|
}
|
||||||
if (typeof settings.sttServerUrl === 'string' && settings.sttServerUrl !== configStore.sttServerUrl) {
|
if (typeof settings.sttServerUrl === 'string' && settings.sttServerUrl !== configStore.sttServerUrl) {
|
||||||
@@ -623,18 +623,12 @@ const applyDesktopUiPreferences = (settings: DesktopSettings) => {
|
|||||||
if (typeof settings.sttModel === 'string' && settings.sttModel !== configStore.sttModel) {
|
if (typeof settings.sttModel === 'string' && settings.sttModel !== configStore.sttModel) {
|
||||||
nextConfigState.sttModel = settings.sttModel;
|
nextConfigState.sttModel = settings.sttModel;
|
||||||
}
|
}
|
||||||
|
if (typeof settings.sttLocalModel === 'string' && settings.sttLocalModel !== configStore.sttLocalModel) {
|
||||||
|
nextConfigState.sttLocalModel = settings.sttLocalModel;
|
||||||
|
}
|
||||||
if (typeof settings.sttLanguage === 'string' && settings.sttLanguage !== configStore.sttLanguage) {
|
if (typeof settings.sttLanguage === 'string' && settings.sttLanguage !== configStore.sttLanguage) {
|
||||||
nextConfigState.sttLanguage = settings.sttLanguage;
|
nextConfigState.sttLanguage = settings.sttLanguage;
|
||||||
}
|
}
|
||||||
if (typeof settings.sttSilenceThresholdDb === 'number' && Number.isFinite(settings.sttSilenceThresholdDb) && settings.sttSilenceThresholdDb !== configStore.sttSilenceThresholdDb) {
|
|
||||||
nextConfigState.sttSilenceThresholdDb = settings.sttSilenceThresholdDb;
|
|
||||||
}
|
|
||||||
if (typeof settings.sttSilenceHoldMs === 'number' && Number.isFinite(settings.sttSilenceHoldMs) && settings.sttSilenceHoldMs !== configStore.sttSilenceHoldMs) {
|
|
||||||
nextConfigState.sttSilenceHoldMs = settings.sttSilenceHoldMs;
|
|
||||||
}
|
|
||||||
if (typeof settings.sttTranscribeOnStop === 'boolean' && settings.sttTranscribeOnStop !== configStore.sttTranscribeOnStop) {
|
|
||||||
nextConfigState.sttTranscribeOnStop = settings.sttTranscribeOnStop;
|
|
||||||
}
|
|
||||||
if (Object.keys(nextConfigState).length > 0) {
|
if (Object.keys(nextConfigState).length > 0) {
|
||||||
configStoreApi.setState(nextConfigState);
|
configStoreApi.setState(nextConfigState);
|
||||||
}
|
}
|
||||||
@@ -1200,8 +1194,16 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
|
|||||||
if (typeof candidate.responseStyleCustomInstructions === 'string') {
|
if (typeof candidate.responseStyleCustomInstructions === 'string') {
|
||||||
result.responseStyleCustomInstructions = candidate.responseStyleCustomInstructions;
|
result.responseStyleCustomInstructions = candidate.responseStyleCustomInstructions;
|
||||||
}
|
}
|
||||||
if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'server') {
|
if (typeof candidate.dictationEnabled === 'boolean') {
|
||||||
|
result.dictationEnabled = candidate.dictationEnabled;
|
||||||
|
}
|
||||||
|
if (candidate.sttProvider === 'local' || candidate.sttProvider === 'openai-compatible') {
|
||||||
result.sttProvider = candidate.sttProvider;
|
result.sttProvider = candidate.sttProvider;
|
||||||
|
} else if (candidate.sttProvider === 'server') {
|
||||||
|
// Legacy provider migration: 'server' was the OpenAI-compatible endpoint.
|
||||||
|
result.sttProvider = 'openai-compatible';
|
||||||
|
} else if (candidate.sttProvider === 'browser' || candidate.sttProvider === 'wasm') {
|
||||||
|
result.sttProvider = 'local';
|
||||||
}
|
}
|
||||||
if (typeof candidate.sttServerUrl === 'string') {
|
if (typeof candidate.sttServerUrl === 'string') {
|
||||||
result.sttServerUrl = candidate.sttServerUrl.trim();
|
result.sttServerUrl = candidate.sttServerUrl.trim();
|
||||||
@@ -1209,18 +1211,12 @@ const sanitizeWebSettings = (payload: unknown): DesktopSettings | null => {
|
|||||||
if (typeof candidate.sttModel === 'string') {
|
if (typeof candidate.sttModel === 'string') {
|
||||||
result.sttModel = candidate.sttModel.trim();
|
result.sttModel = candidate.sttModel.trim();
|
||||||
}
|
}
|
||||||
|
if (typeof candidate.sttLocalModel === 'string') {
|
||||||
|
result.sttLocalModel = candidate.sttLocalModel.trim();
|
||||||
|
}
|
||||||
if (typeof candidate.sttLanguage === 'string') {
|
if (typeof candidate.sttLanguage === 'string') {
|
||||||
result.sttLanguage = candidate.sttLanguage.trim();
|
result.sttLanguage = candidate.sttLanguage.trim();
|
||||||
}
|
}
|
||||||
if (typeof candidate.sttSilenceThresholdDb === 'number' && Number.isFinite(candidate.sttSilenceThresholdDb)) {
|
|
||||||
result.sttSilenceThresholdDb = candidate.sttSilenceThresholdDb;
|
|
||||||
}
|
|
||||||
if (typeof candidate.sttSilenceHoldMs === 'number' && Number.isFinite(candidate.sttSilenceHoldMs)) {
|
|
||||||
result.sttSilenceHoldMs = candidate.sttSilenceHoldMs;
|
|
||||||
}
|
|
||||||
if (typeof candidate.sttTranscribeOnStop === 'boolean') {
|
|
||||||
result.sttTranscribeOnStop = candidate.sttTranscribeOnStop;
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -678,22 +678,16 @@ const SETTINGS_SEARCH_ITEMS: readonly SettingsSearchItem[] = [
|
|||||||
keywords: ['keyboard', 'hotkeys', 'bindings'],
|
keywords: ['keyboard', 'hotkeys', 'bindings'],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'voice.voice-setup',
|
id: 'voice.playback',
|
||||||
page: 'voice',
|
page: 'voice',
|
||||||
titleKey: 'settings.voice.page.section.voiceSetup',
|
titleKey: 'settings.voice.page.section.playbackAndSummary',
|
||||||
keywords: ['tts', 'voice mode', 'provider', 'speech rate', 'speech pitch', 'speech volume', 'language'],
|
keywords: ['tts', 'read aloud', 'voice', 'provider', 'speech rate', 'speech pitch', 'speech volume', 'tts input mode', 'markdown'],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'voice.speech-recognition',
|
id: 'voice.speech-recognition',
|
||||||
page: 'voice',
|
page: 'voice',
|
||||||
titleKey: 'settings.voice.page.section.speechRecognition',
|
titleKey: 'settings.voice.page.section.speechRecognition',
|
||||||
keywords: ['stt', 'transcribe', 'whisper', 'microphone', 'silence threshold'],
|
keywords: ['stt', 'dictation', 'voice input', 'transcribe', 'whisper', 'parakeet', 'microphone'],
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'voice.playback',
|
|
||||||
page: 'voice',
|
|
||||||
titleKey: 'settings.voice.page.section.playbackAndSummary',
|
|
||||||
keywords: ['read aloud', 'tts input mode', 'summary', 'markdown'],
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: 'tunnel.provider',
|
id: 'tunnel.provider',
|
||||||
|
|||||||
@@ -327,6 +327,13 @@ const SHORTCUT_ACTIONS: ReadonlyArray<ShortcutAction> = [
|
|||||||
description: 'Toggle focus mode for the chat input',
|
description: 'Toggle focus mode for the chat input',
|
||||||
customizable: true,
|
customizable: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: 'toggle_dictation',
|
||||||
|
defaultCombo: 'mod+alt+v',
|
||||||
|
label: 'Voice input',
|
||||||
|
description: 'Start dictation; press again to confirm and insert the transcript',
|
||||||
|
customizable: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: 'abort_run',
|
id: 'abort_run',
|
||||||
defaultCombo: 'escape',
|
defaultCombo: 'escape',
|
||||||
|
|||||||
@@ -1,397 +0,0 @@
|
|||||||
/**
|
|
||||||
* Audio Stream Service
|
|
||||||
*
|
|
||||||
* Captures microphone audio using MediaRecorder, detects utterance boundaries
|
|
||||||
* via an AnalyserNode-based silence detector (VAD), then POSTs each utterance
|
|
||||||
* as a raw audio blob to the OpenChamber server's /api/stt/transcribe endpoint.
|
|
||||||
*
|
|
||||||
* Mimics the BrowserVoiceService.startListening interface so useBrowserVoice
|
|
||||||
* can swap providers without changing its internal logic.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* audioStreamService.configure({ baseURL: 'http://localhost:8001/v1', model: 'whisper-1' });
|
|
||||||
* audioStreamService.startListening('en', (text, isFinal) => {
|
|
||||||
* if (isFinal) console.log('transcript:', text);
|
|
||||||
* });
|
|
||||||
* audioStreamService.stopListening();
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
|
||||||
|
|
||||||
type SpeechResultCallback = (text: string, isFinal: boolean) => void;
|
|
||||||
type ErrorCallback = (error: string) => void;
|
|
||||||
|
|
||||||
interface AudioStreamConfig {
|
|
||||||
/** Base URL of the OpenAI-compatible STT server (e.g. http://localhost:8001/v1) */
|
|
||||||
baseURL: string;
|
|
||||||
/** Whisper-compatible model name */
|
|
||||||
model: string;
|
|
||||||
/** Optional BCP-47 language hint (e.g. 'en'). Empty string = auto-detect. */
|
|
||||||
language?: string;
|
|
||||||
/**
|
|
||||||
* Silence threshold in dB below which audio is considered silence.
|
|
||||||
* Lower (more negative) = only very quiet audio counts as silence.
|
|
||||||
* Default: -45
|
|
||||||
*/
|
|
||||||
silenceThresholdDb?: number;
|
|
||||||
/**
|
|
||||||
* How long continuous silence must last (ms) before the utterance is finalised.
|
|
||||||
* Default: 1500
|
|
||||||
*/
|
|
||||||
silenceHoldMs?: number;
|
|
||||||
/** Optional API key for the STT server. */
|
|
||||||
apiKey?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
// How often (ms) the VAD samples the analyser
|
|
||||||
const VAD_POLL_MS = 80;
|
|
||||||
// Minimum audio duration (ms) to bother uploading (avoids blank clips)
|
|
||||||
const MIN_UTTERANCE_MS = 300;
|
|
||||||
|
|
||||||
class AudioStreamService {
|
|
||||||
private stream: MediaStream | null = null;
|
|
||||||
private mediaRecorder: MediaRecorder | null = null;
|
|
||||||
private audioContext: AudioContext | null = null;
|
|
||||||
private analyser: AnalyserNode | null = null;
|
|
||||||
private vadTimer: ReturnType<typeof setInterval> | null = null;
|
|
||||||
private chunks: Blob[] = [];
|
|
||||||
private recordingStartMs = 0;
|
|
||||||
private isActive = false;
|
|
||||||
private isSpeaking = false;
|
|
||||||
private silenceSince: number | null = null;
|
|
||||||
private onResult: SpeechResultCallback | null = null;
|
|
||||||
private onError: ErrorCallback | null = null;
|
|
||||||
private finishResolver: (() => void) | null = null;
|
|
||||||
private lang = 'en';
|
|
||||||
|
|
||||||
// Configurable parameters
|
|
||||||
private cfg: Required<AudioStreamConfig> = {
|
|
||||||
baseURL: '',
|
|
||||||
model: 'deepdml/faster-whisper-large-v3-turbo-ct2',
|
|
||||||
language: '',
|
|
||||||
silenceThresholdDb: -45,
|
|
||||||
silenceHoldMs: 1500,
|
|
||||||
apiKey: '',
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Update service configuration. Can be called before or after startListening. */
|
|
||||||
configure(config: AudioStreamConfig): void {
|
|
||||||
this.cfg = {
|
|
||||||
silenceThresholdDb: -45,
|
|
||||||
silenceHoldMs: 1500,
|
|
||||||
language: '',
|
|
||||||
apiKey: '',
|
|
||||||
...config,
|
|
||||||
};
|
|
||||||
this.cfg.apiKey = config.apiKey ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Whether the browser supports the required APIs. */
|
|
||||||
isSupported(): boolean {
|
|
||||||
return (
|
|
||||||
typeof window !== 'undefined' &&
|
|
||||||
typeof navigator !== 'undefined' &&
|
|
||||||
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
|
|
||||||
typeof window.MediaRecorder !== 'undefined' &&
|
|
||||||
typeof window.AudioContext !== 'undefined'
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start listening. Requests microphone access if not already held.
|
|
||||||
* Calls onResult(text, true) for each completed utterance.
|
|
||||||
*/
|
|
||||||
async startListening(
|
|
||||||
lang: string,
|
|
||||||
onResult: SpeechResultCallback,
|
|
||||||
onError?: ErrorCallback
|
|
||||||
): Promise<void> {
|
|
||||||
if (this.isActive) {
|
|
||||||
this.stopListening();
|
|
||||||
}
|
|
||||||
|
|
||||||
this.lang = lang;
|
|
||||||
this.onResult = onResult;
|
|
||||||
this.onError = onError ?? null;
|
|
||||||
this.isActive = true;
|
|
||||||
|
|
||||||
try {
|
|
||||||
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
|
||||||
} catch (err) {
|
|
||||||
this.isActive = false;
|
|
||||||
const msg = err instanceof Error ? err.message : 'Microphone access denied';
|
|
||||||
onError?.(msg);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this._setupAudioContext();
|
|
||||||
this._startRecorder();
|
|
||||||
this._startVAD();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Stop listening and clean up all resources. */
|
|
||||||
stopListening(): void {
|
|
||||||
this._stopVAD();
|
|
||||||
this._stopRecorder();
|
|
||||||
this._cleanupAfterStop(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
async finishListening(): Promise<void> {
|
|
||||||
if (!this.isActive) return;
|
|
||||||
|
|
||||||
this._stopVAD();
|
|
||||||
this.isSpeaking = false;
|
|
||||||
this.silenceSince = null;
|
|
||||||
|
|
||||||
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
|
|
||||||
this._cleanupAfterStop(true);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await new Promise<void>((resolve) => {
|
|
||||||
this.finishResolver = resolve;
|
|
||||||
this._finaliseUtterance(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
this._cleanupAfterStop(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Whether currently listening. */
|
|
||||||
getIsListening(): boolean {
|
|
||||||
return this.isActive;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Private helpers ──────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
private _setupAudioContext(): void {
|
|
||||||
if (!this.stream) return;
|
|
||||||
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
|
|
||||||
this.audioContext = new AudioContextClass();
|
|
||||||
const source = this.audioContext.createMediaStreamSource(this.stream);
|
|
||||||
this.analyser = this.audioContext.createAnalyser();
|
|
||||||
this.analyser.fftSize = 512;
|
|
||||||
source.connect(this.analyser);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _teardownAudioContext(): void {
|
|
||||||
try {
|
|
||||||
this.audioContext?.close();
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
this.audioContext = null;
|
|
||||||
this.analyser = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private _startRecorder(): void {
|
|
||||||
if (!this.stream) return;
|
|
||||||
|
|
||||||
const mimeType = this._pickMimeType();
|
|
||||||
const options: MediaRecorderOptions = {};
|
|
||||||
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
|
|
||||||
options.mimeType = mimeType;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.mediaRecorder = new MediaRecorder(this.stream, options);
|
|
||||||
this.chunks = [];
|
|
||||||
this.recordingStartMs = Date.now();
|
|
||||||
|
|
||||||
this.mediaRecorder.ondataavailable = (e) => {
|
|
||||||
if (e.data && e.data.size > 0) {
|
|
||||||
this.chunks.push(e.data);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
this.mediaRecorder.onstop = () => {
|
|
||||||
const blobs = this.chunks.splice(0);
|
|
||||||
const durationMs = Date.now() - this.recordingStartMs;
|
|
||||||
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
|
|
||||||
this.finishResolver?.();
|
|
||||||
this.finishResolver = null;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const mType = blobs[0].type || mimeType || 'audio/webm';
|
|
||||||
const blob = new Blob(blobs, { type: mType });
|
|
||||||
void this._upload(blob, mType).finally(() => {
|
|
||||||
this.finishResolver?.();
|
|
||||||
this.finishResolver = null;
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
// Collect data every 250 ms so we don't lose the tail on stop()
|
|
||||||
this.mediaRecorder.start(250);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _stopRecorder(): void {
|
|
||||||
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
|
|
||||||
try {
|
|
||||||
this.mediaRecorder.stop();
|
|
||||||
} catch {
|
|
||||||
this.finishResolver?.();
|
|
||||||
this.finishResolver = null;
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.mediaRecorder = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private _releaseStream(): void {
|
|
||||||
if (this.stream) {
|
|
||||||
this.stream.getTracks().forEach((t) => t.stop());
|
|
||||||
this.stream = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _startVAD(): void {
|
|
||||||
this._stopVAD();
|
|
||||||
this.silenceSince = null;
|
|
||||||
this.isSpeaking = false;
|
|
||||||
|
|
||||||
this.vadTimer = setInterval(() => {
|
|
||||||
if (!this.isActive || !this.analyser) return;
|
|
||||||
const db = this._getRmsDb();
|
|
||||||
const isSilent = db < this.cfg.silenceThresholdDb;
|
|
||||||
|
|
||||||
if (!isSilent) {
|
|
||||||
// Audio detected
|
|
||||||
this.silenceSince = null;
|
|
||||||
if (!this.isSpeaking) {
|
|
||||||
this.isSpeaking = true;
|
|
||||||
// Restart recorder to capture from the start of speech
|
|
||||||
if (this.mediaRecorder?.state === 'recording') {
|
|
||||||
this.recordingStartMs = Date.now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Silence detected
|
|
||||||
if (this.isSpeaking) {
|
|
||||||
if (this.silenceSince === null) {
|
|
||||||
this.silenceSince = Date.now();
|
|
||||||
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
|
|
||||||
// End of utterance — stop recorder (triggers onstop → upload)
|
|
||||||
this.isSpeaking = false;
|
|
||||||
this.silenceSince = null;
|
|
||||||
this._finaliseUtterance(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, VAD_POLL_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _stopVAD(): void {
|
|
||||||
if (this.vadTimer !== null) {
|
|
||||||
clearInterval(this.vadTimer);
|
|
||||||
this.vadTimer = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _cleanupAfterStop(clearChunks: boolean): void {
|
|
||||||
const pendingResolver = this.finishResolver;
|
|
||||||
this.isActive = false;
|
|
||||||
this.finishResolver = null;
|
|
||||||
this.mediaRecorder = null;
|
|
||||||
this._teardownAudioContext();
|
|
||||||
this._releaseStream();
|
|
||||||
if (clearChunks) {
|
|
||||||
this.chunks = [];
|
|
||||||
}
|
|
||||||
this.isSpeaking = false;
|
|
||||||
this.silenceSince = null;
|
|
||||||
this.onResult = null;
|
|
||||||
this.onError = null;
|
|
||||||
pendingResolver?.();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Stop the current recorder to flush the utterance, optionally restarting for the next one. */
|
|
||||||
private _finaliseUtterance(restart: boolean): void {
|
|
||||||
if (!this.isActive) return;
|
|
||||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
|
||||||
this.mediaRecorder.stop();
|
|
||||||
}
|
|
||||||
if (!restart) return;
|
|
||||||
|
|
||||||
// Restart recorder for the next utterance after a short delay
|
|
||||||
// (MediaRecorder.onstop fires asynchronously; we wait for it to complete)
|
|
||||||
setTimeout(() => {
|
|
||||||
if (this.isActive && this.stream) {
|
|
||||||
this._startRecorder();
|
|
||||||
}
|
|
||||||
}, 100);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Compute RMS of current analyser frame in dBFS. */
|
|
||||||
private _getRmsDb(): number {
|
|
||||||
if (!this.analyser) return -Infinity;
|
|
||||||
const buf = new Float32Array(this.analyser.fftSize);
|
|
||||||
this.analyser.getFloatTimeDomainData(buf);
|
|
||||||
let sumSq = 0;
|
|
||||||
for (const s of buf) sumSq += s * s;
|
|
||||||
const rms = Math.sqrt(sumSq / buf.length);
|
|
||||||
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** POST utterance blob to server, call onResult with transcript. */
|
|
||||||
private async _upload(blob: Blob, mimeType: string): Promise<void> {
|
|
||||||
if (!this.onResult) return;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const headers: Record<string, string> = {
|
|
||||||
'Content-Type': mimeType,
|
|
||||||
'X-Base-URL': this.cfg.baseURL,
|
|
||||||
'X-Model': this.cfg.model,
|
|
||||||
};
|
|
||||||
if (this.cfg.apiKey) {
|
|
||||||
headers['Authorization'] = `Bearer ${this.cfg.apiKey}`;
|
|
||||||
}
|
|
||||||
if (this.cfg.language) {
|
|
||||||
headers['X-Language'] = this.cfg.language;
|
|
||||||
} else if (this.lang && this.lang !== 'auto') {
|
|
||||||
// Use BCP-47 base language code (e.g. 'en' from 'en-US')
|
|
||||||
const baseLang = this.lang.split('-')[0];
|
|
||||||
headers['X-Language'] = baseLang;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await runtimeFetch('/api/stt/transcribe', {
|
|
||||||
method: 'POST',
|
|
||||||
headers,
|
|
||||||
body: blob,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
|
||||||
throw new Error(errData.error ?? `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
|
||||||
const transcript: string = (data.transcript ?? '').trim();
|
|
||||||
if (transcript) {
|
|
||||||
this.onResult(transcript, true);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
if (!this.isActive) return; // Stopped — ignore
|
|
||||||
const msg = err instanceof Error ? err.message : 'Transcription upload failed';
|
|
||||||
console.error('[AudioStreamService] Upload error:', msg);
|
|
||||||
this.onError?.(msg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Pick the best supported MIME type for MediaRecorder. */
|
|
||||||
private _pickMimeType(): string {
|
|
||||||
const candidates = [
|
|
||||||
'audio/webm;codecs=opus',
|
|
||||||
'audio/webm',
|
|
||||||
'audio/ogg;codecs=opus',
|
|
||||||
'audio/ogg',
|
|
||||||
'audio/mp4',
|
|
||||||
];
|
|
||||||
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
|
|
||||||
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
|
|
||||||
}
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const audioStreamService = new AudioStreamService();
|
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
/**
|
|
||||||
* Context formatters for voice-native output
|
|
||||||
* Formats session events (messages, permissions, ready events) into natural language
|
|
||||||
* for the ElevenLabs voice agent to speak aloud.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* import { formatMessage, formatPermissionRequest } from '@/lib/voice';
|
|
||||||
*
|
|
||||||
* const voiceText = formatMessage({ role: 'assistant', content: 'Hello!' });
|
|
||||||
* // Returns: "Claude Code: Hello!"
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { VOICE_CONFIG } from "./voiceConfig";
|
|
||||||
|
|
||||||
/** Message type for voice formatting */
|
|
||||||
export interface VoiceMessage {
|
|
||||||
role: string;
|
|
||||||
content: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a single message for voice output
|
|
||||||
* - Assistant messages: Code blocks replaced with "[code block]", prefixed with "Claude Code: "
|
|
||||||
* - User messages: Prefixed with "User: "
|
|
||||||
* - Other roles: Returns null (not spoken)
|
|
||||||
*
|
|
||||||
* @param message - The message to format
|
|
||||||
* @returns Formatted text for voice, or null if should not be spoken
|
|
||||||
*/
|
|
||||||
function formatMessage(message: VoiceMessage): string | null {
|
|
||||||
// Handle edge cases
|
|
||||||
if (!message || typeof message.content !== "string") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const content = message.content.trim();
|
|
||||||
if (!content) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (message.role === "assistant") {
|
|
||||||
// Replace code blocks with description (don't read code aloud)
|
|
||||||
const textOnly = content.replace(/```[\s\S]*?```/g, "[code block]");
|
|
||||||
return `Claude Code: ${textOnly}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (message.role === "user") {
|
|
||||||
return `User: ${content}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip system, tool, and other roles for voice
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format multiple new messages for voice output
|
|
||||||
* - Maps messages through formatMessage
|
|
||||||
* - Filters out nulls (unspoken roles)
|
|
||||||
* - Joins with newlines
|
|
||||||
*
|
|
||||||
* @param sessionId - The session ID (for future use/debugging)
|
|
||||||
* @param messages - Array of messages to format
|
|
||||||
* @returns Formatted text for voice, or null if no speakable messages
|
|
||||||
*/
|
|
||||||
export function formatNewMessages(
|
|
||||||
sessionId: string,
|
|
||||||
messages: VoiceMessage[]
|
|
||||||
): string | null {
|
|
||||||
// Handle edge cases
|
|
||||||
if (!Array.isArray(messages) || messages.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log(`[Voice] Formatting ${messages.length} messages for session ${sessionId}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Format each message and filter out nulls
|
|
||||||
const formattedMessages = messages
|
|
||||||
.map(formatMessage)
|
|
||||||
.filter((msg): msg is string => msg !== null);
|
|
||||||
|
|
||||||
if (formattedMessages.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return formattedMessages.join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a permission request for voice announcement
|
|
||||||
* - Per CONTEXT.md: Only tool name, not arguments (LIMITED_TOOL_CALLS)
|
|
||||||
* - Prompts user to say "allow" or "deny"
|
|
||||||
*
|
|
||||||
* @param sessionId - The session ID
|
|
||||||
* @param requestId - The permission request ID
|
|
||||||
* @param toolName - Name of the tool requesting permission
|
|
||||||
* @param toolArgs - Tool arguments (not included in voice output per config)
|
|
||||||
* @returns Formatted permission request for voice
|
|
||||||
*/
|
|
||||||
export function formatPermissionRequest(
|
|
||||||
sessionId: string,
|
|
||||||
requestId: string,
|
|
||||||
toolName: string,
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
||||||
toolArgs: unknown
|
|
||||||
): string {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log(`[Voice] Formatting permission request ${requestId} for session ${sessionId}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Per VOICE_CONFIG.LIMITED_TOOL_CALLS, we don't include toolArgs in voice output
|
|
||||||
return `Claude Code is requesting permission to use ${toolName}. Say "allow" or "deny".`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a ready event for voice announcement
|
|
||||||
* - Indicates the AI has finished working and is ready for next instruction
|
|
||||||
*
|
|
||||||
* @param sessionId - The session ID
|
|
||||||
* @returns Formatted ready event for voice
|
|
||||||
*/
|
|
||||||
export function formatReadyEvent(sessionId: string): string {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log(`[Voice] Formatting ready event for session ${sessionId}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return "Claude Code finished working. Ready for next instruction.";
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
/**
|
|
||||||
* Voice module barrel export
|
|
||||||
* Provides a clean import path for voice session hooks.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* import { voiceHooks } from '@/lib/voice';
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Voice session registry (from voiceSession.ts)
|
|
||||||
export {
|
|
||||||
isVoiceSessionStarted,
|
|
||||||
} from "./voiceSession";
|
|
||||||
|
|
||||||
// Voice hooks for session-to-voice event routing (from voiceHooks.ts)
|
|
||||||
export { voiceHooks } from "./voiceHooks";
|
|
||||||
@@ -6,15 +6,21 @@
|
|||||||
export function sanitizeForTTS(text: string): string {
|
export function sanitizeForTTS(text: string): string {
|
||||||
if (!text) return '';
|
if (!text) return '';
|
||||||
return text
|
return text
|
||||||
// Remove code blocks
|
// Remove fenced code blocks entirely (multi-line code is unreadable),
|
||||||
|
// but keep inline-code CONTENT and only strip the backticks: agents
|
||||||
|
// routinely inline meaningful words ("You are on `main`").
|
||||||
.replace(/```[\s\S]*?```/g, '')
|
.replace(/```[\s\S]*?```/g, '')
|
||||||
.replace(/`[^`]*`/g, '')
|
.replace(/`([^`\n]*)`/g, '$1')
|
||||||
// Remove markdown formatting
|
// Remove markdown formatting
|
||||||
.replace(/[*_~#]/g, '')
|
.replace(/[*_~#]/g, '')
|
||||||
// Remove URLs
|
// Remove URLs
|
||||||
.replace(/https?:\/\/[^\s]+/g, '')
|
.replace(/https?:\/\/[^\s]+/g, '')
|
||||||
// Remove file paths
|
// Remove absolute file paths (leading slash, one or more segments).
|
||||||
.replace(/\/[\w\-./]+/g, '')
|
// Deliberately NOT matching interword slashes: "iOS/Android" and
|
||||||
|
// "origin/main" are speech, not paths.
|
||||||
|
.replace(/(^|\s)\/(?:[\w.-]+\/)*[\w.-]+/g, '$1')
|
||||||
|
// Read remaining interword slashes out loud ("iOS slash Android").
|
||||||
|
.replace(/([\w.])\/([\w.])/g, '$1 slash $2')
|
||||||
// Remove shell-like patterns
|
// Remove shell-like patterns
|
||||||
.replace(/^\s*[$#>]\s*/gm, '')
|
.replace(/^\s*[$#>]\s*/gm, '')
|
||||||
// Remove brackets and special chars
|
// Remove brackets and special chars
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
/**
|
|
||||||
* Static voice context configuration
|
|
||||||
* Controls voice behavior and feature flags for the ElevenLabs voice agent
|
|
||||||
*/
|
|
||||||
export const VOICE_CONFIG = {
|
|
||||||
/** Disable all tool call information from being sent to voice context */
|
|
||||||
DISABLE_TOOL_CALLS: false,
|
|
||||||
|
|
||||||
/** Send only tool names and descriptions, exclude arguments */
|
|
||||||
LIMITED_TOOL_CALLS: true,
|
|
||||||
|
|
||||||
/** Disable permission request forwarding */
|
|
||||||
DISABLE_PERMISSION_REQUESTS: false,
|
|
||||||
|
|
||||||
/** Disable session online/offline notifications */
|
|
||||||
DISABLE_SESSION_STATUS: true,
|
|
||||||
|
|
||||||
/** Disable message forwarding */
|
|
||||||
DISABLE_MESSAGES: false,
|
|
||||||
|
|
||||||
/** Disable session focus notifications */
|
|
||||||
DISABLE_SESSION_FOCUS: false,
|
|
||||||
|
|
||||||
/** Disable ready event notifications */
|
|
||||||
DISABLE_READY_EVENTS: false,
|
|
||||||
|
|
||||||
/** Maximum number of messages to include in session history */
|
|
||||||
MAX_HISTORY_MESSAGES: 50,
|
|
||||||
|
|
||||||
/** Enable debug logging for voice context updates */
|
|
||||||
ENABLE_DEBUG_LOGGING: true,
|
|
||||||
} as const;
|
|
||||||
@@ -1,125 +0,0 @@
|
|||||||
/**
|
|
||||||
* Voice hooks for session-to-voice event routing
|
|
||||||
* Routes session events (messages, permissions, ready events) to the ElevenLabs
|
|
||||||
* voice agent via contextual updates.
|
|
||||||
*
|
|
||||||
* This module provides hooks that can be called when session events occur,
|
|
||||||
* using the voice session registry from voiceSession.ts.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```typescript
|
|
||||||
* import { voiceHooks } from '@/lib/voice';
|
|
||||||
*
|
|
||||||
* // Route session messages to voice
|
|
||||||
* voiceHooks.onMessages(sessionId, messages);
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { VOICE_CONFIG } from "./voiceConfig";
|
|
||||||
import {
|
|
||||||
formatNewMessages,
|
|
||||||
formatPermissionRequest,
|
|
||||||
formatReadyEvent,
|
|
||||||
type VoiceMessage,
|
|
||||||
} from "./contextFormatters";
|
|
||||||
import { getVoiceSession, isVoiceSessionStarted } from "./voiceSession";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Report a contextual update to the voice session
|
|
||||||
* Internal helper that checks preconditions and handles errors
|
|
||||||
*
|
|
||||||
* @param update - The text update to send, or null/undefined to skip
|
|
||||||
*/
|
|
||||||
function reportContextualUpdate(update: string | null | undefined): void {
|
|
||||||
// Skip empty/null/undefined updates
|
|
||||||
if (!update || update.trim().length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip if no voice session or not started
|
|
||||||
const voiceSession = getVoiceSession();
|
|
||||||
if (!voiceSession || !isVoiceSessionStarted()) {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log("[Voice] Skipping contextual update - no active session");
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log("[Voice] Sending contextual update:", update.substring(0, 100));
|
|
||||||
}
|
|
||||||
voiceSession.sendContextualUpdate(update);
|
|
||||||
} catch (error) {
|
|
||||||
// Log error but don't throw - voice updates shouldn't break the app
|
|
||||||
console.error("[Voice] Failed to send contextual update:", error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Voice hooks - exported functions to route session events to voice
|
|
||||||
*
|
|
||||||
* These hooks should be called when corresponding session events occur.
|
|
||||||
* They respect VOICE_CONFIG feature flags to enable/disable specific
|
|
||||||
* event types.
|
|
||||||
*/
|
|
||||||
export const voiceHooks = {
|
|
||||||
/**
|
|
||||||
* Called when new messages arrive in the session
|
|
||||||
* Formats and sends messages to voice agent (if not disabled)
|
|
||||||
*
|
|
||||||
* @param sessionId - The session ID
|
|
||||||
* @param messages - Array of messages to format and send
|
|
||||||
*/
|
|
||||||
onMessages(sessionId: string, messages: VoiceMessage[]): void {
|
|
||||||
if (VOICE_CONFIG.DISABLE_MESSAGES) {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log("[Voice] Message forwarding disabled");
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
reportContextualUpdate(formatNewMessages(sessionId, messages));
|
|
||||||
},
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Called when a permission request is made
|
|
||||||
* Announces the permission request to the voice agent (if not disabled)
|
|
||||||
*
|
|
||||||
* @param sessionId - The session ID
|
|
||||||
* @param requestId - The permission request ID
|
|
||||||
* @param toolName - Name of the tool requesting permission
|
|
||||||
* @param toolArgs - Arguments for the tool (not sent to voice per LIMITED_TOOL_CALLS)
|
|
||||||
*/
|
|
||||||
onPermissionRequested(
|
|
||||||
sessionId: string,
|
|
||||||
requestId: string,
|
|
||||||
toolName: string,
|
|
||||||
toolArgs: unknown
|
|
||||||
): void {
|
|
||||||
if (VOICE_CONFIG.DISABLE_PERMISSION_REQUESTS) {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log("[Voice] Permission request forwarding disabled");
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
reportContextualUpdate(
|
|
||||||
formatPermissionRequest(sessionId, requestId, toolName, toolArgs)
|
|
||||||
);
|
|
||||||
},
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Called when the AI is ready for the next instruction
|
|
||||||
* Announces ready state to the voice agent (if not disabled)
|
|
||||||
*
|
|
||||||
* @param sessionId - The session ID
|
|
||||||
*/
|
|
||||||
onReady(sessionId: string): void {
|
|
||||||
if (VOICE_CONFIG.DISABLE_READY_EVENTS) {
|
|
||||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
|
||||||
console.log("[Voice] Ready event forwarding disabled");
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
reportContextualUpdate(formatReadyEvent(sessionId));
|
|
||||||
},
|
|
||||||
};
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
/**
|
|
||||||
* Voice session interface
|
|
||||||
* Used for type safety without importing ReturnType from SDK
|
|
||||||
*/
|
|
||||||
interface VoiceSession {
|
|
||||||
sendContextualUpdate: (text: string) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Global storage for the active voice session.
|
|
||||||
* Used by voiceHooks to send contextual updates to the voice agent.
|
|
||||||
*/
|
|
||||||
const activeVoiceSession: VoiceSession | null = null;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the currently registered voice session.
|
|
||||||
* Used by voiceHooks to send contextual updates.
|
|
||||||
*/
|
|
||||||
export function getVoiceSession(): VoiceSession | null {
|
|
||||||
return activeVoiceSession;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a voice session is currently active.
|
|
||||||
*/
|
|
||||||
export function isVoiceSessionStarted(): boolean {
|
|
||||||
return activeVoiceSession !== null;
|
|
||||||
}
|
|
||||||
@@ -1,555 +0,0 @@
|
|||||||
/**
|
|
||||||
* WASM Speech-to-Text Service
|
|
||||||
*
|
|
||||||
* Local Whisper transcription via Transformers.js (ONNX Runtime Web).
|
|
||||||
* Captures microphone audio, detects utterance boundaries via silence-based
|
|
||||||
* VAD, then transcribes each utterance locally — no cloud API required.
|
|
||||||
*
|
|
||||||
* Works in Electron and all modern browsers that support Web Audio API.
|
|
||||||
* First use downloads a Whisper model (~40–166 MB, cached).
|
|
||||||
*/
|
|
||||||
|
|
||||||
export type WasmModelStatus =
|
|
||||||
| { state: 'unloaded' }
|
|
||||||
| { state: 'downloading'; progress: number }
|
|
||||||
| { state: 'loading' }
|
|
||||||
| { state: 'ready' }
|
|
||||||
| { state: 'error'; error: string };
|
|
||||||
|
|
||||||
export interface WasmModelInfo {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
size: string;
|
|
||||||
languages: string;
|
|
||||||
description: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const WASM_MODELS: WasmModelInfo[] = [
|
|
||||||
{
|
|
||||||
id: 'Xenova/whisper-tiny.en',
|
|
||||||
name: 'Whisper Tiny (EN)',
|
|
||||||
size: '~39 MB',
|
|
||||||
languages: 'English',
|
|
||||||
description: 'Fastest, lowest accuracy. Good for quick dictation.',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'Xenova/whisper-base.en',
|
|
||||||
name: 'Whisper Base (EN)',
|
|
||||||
size: '~73 MB',
|
|
||||||
languages: 'English',
|
|
||||||
description: 'Balanced speed and accuracy. Default for English.',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'Xenova/whisper-small.en',
|
|
||||||
name: 'Whisper Small (EN)',
|
|
||||||
size: '~166 MB',
|
|
||||||
languages: 'English',
|
|
||||||
description: 'Higher accuracy, slower. Best for noisy environments.',
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
type SpeechResultCallback = (text: string, isFinal: boolean) => void;
|
|
||||||
type ErrorCallback = (error: string) => void;
|
|
||||||
|
|
||||||
const VAD_POLL_MS = 80;
|
|
||||||
const MIN_UTTERANCE_MS = 300;
|
|
||||||
const WHISPER_SAMPLE_RATE = 16000;
|
|
||||||
|
|
||||||
interface WasmSttConfig {
|
|
||||||
silenceThresholdDb?: number;
|
|
||||||
silenceHoldMs?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
class WasmSttService {
|
|
||||||
private transcriber: unknown = null;
|
|
||||||
private worker: Worker | null = null;
|
|
||||||
private modelStatus: WasmModelStatus = { state: 'unloaded' };
|
|
||||||
private currentModelId: string | null = null;
|
|
||||||
|
|
||||||
private stream: MediaStream | null = null;
|
|
||||||
private mediaRecorder: MediaRecorder | null = null;
|
|
||||||
private audioContext: AudioContext | null = null;
|
|
||||||
private analyser: AnalyserNode | null = null;
|
|
||||||
private vadTimer: ReturnType<typeof setInterval> | null = null;
|
|
||||||
private chunks: Blob[] = [];
|
|
||||||
private recordingStartMs = 0;
|
|
||||||
private isActive = false;
|
|
||||||
private isSpeaking = false;
|
|
||||||
private silenceSince: number | null = null;
|
|
||||||
private onResult: SpeechResultCallback | null = null;
|
|
||||||
private onError: ErrorCallback | null = null;
|
|
||||||
private finishResolver: (() => void) | null = null;
|
|
||||||
private lang = 'en';
|
|
||||||
|
|
||||||
private cfg: Required<WasmSttConfig> = {
|
|
||||||
silenceThresholdDb: -45,
|
|
||||||
silenceHoldMs: 1500,
|
|
||||||
};
|
|
||||||
|
|
||||||
public onModelStatusChange: ((status: WasmModelStatus) => void) | null = null;
|
|
||||||
|
|
||||||
configure(config: WasmSttConfig): void {
|
|
||||||
this.cfg = { ...this.cfg, ...config };
|
|
||||||
}
|
|
||||||
|
|
||||||
isSupported(): boolean {
|
|
||||||
return (
|
|
||||||
typeof window !== 'undefined' &&
|
|
||||||
typeof navigator !== 'undefined' &&
|
|
||||||
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
|
|
||||||
typeof window.MediaRecorder !== 'undefined' &&
|
|
||||||
typeof window.AudioContext !== 'undefined'
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
getModelStatus(): WasmModelStatus {
|
|
||||||
return this.modelStatus;
|
|
||||||
}
|
|
||||||
|
|
||||||
getCurrentModelId(): string | null {
|
|
||||||
return this.currentModelId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private setModelStatus(status: WasmModelStatus): void {
|
|
||||||
this.modelStatus = status;
|
|
||||||
this.onModelStatusChange?.(status);
|
|
||||||
}
|
|
||||||
|
|
||||||
async loadModel(modelId: string): Promise<void> {
|
|
||||||
if (this.currentModelId === modelId && this.modelStatus.state === 'ready') {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.modelStatus.state === 'downloading' || this.modelStatus.state === 'loading') {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this._terminateWorker();
|
|
||||||
this.transcriber = null;
|
|
||||||
|
|
||||||
this.setModelStatus({ state: 'downloading', progress: 0 });
|
|
||||||
this.currentModelId = modelId;
|
|
||||||
|
|
||||||
// Try Web Worker first — inference off main thread = no UI freeze.
|
|
||||||
try {
|
|
||||||
const WasmWorkerMod = await import('./wasmSttWorker?worker');
|
|
||||||
const WasmWorker = WasmWorkerMod.default as new () => Worker;
|
|
||||||
this.worker = new WasmWorker();
|
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
const timer = setTimeout(() => reject(new Error('Worker init timed out')), 10000);
|
|
||||||
|
|
||||||
this.worker!.onmessage = (e: MessageEvent) => {
|
|
||||||
const data = e.data as { type: string; progress?: number; error?: string; text?: string };
|
|
||||||
if (data.type === 'progress') {
|
|
||||||
this.setModelStatus({ state: 'downloading', progress: data.progress ?? 0 });
|
|
||||||
} else if (data.type === 'loaded') {
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve();
|
|
||||||
} else if (data.type === 'error') {
|
|
||||||
clearTimeout(timer);
|
|
||||||
reject(new Error(data.error ?? 'Worker load failed'));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
this.worker!.onerror = (err) => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
reject(new Error(err.message || 'Worker error'));
|
|
||||||
};
|
|
||||||
|
|
||||||
this.worker!.postMessage({ type: 'load', modelId });
|
|
||||||
});
|
|
||||||
|
|
||||||
this.setModelStatus({ state: 'ready' });
|
|
||||||
return;
|
|
||||||
} catch (err) {
|
|
||||||
console.warn('[WasmStt] Worker failed, using main-thread:', err instanceof Error ? err.message : err);
|
|
||||||
this._terminateWorker();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: main-thread pipeline (causes brief UI freeze during inference).
|
|
||||||
try {
|
|
||||||
const { pipeline, env } = await import('@xenova/transformers');
|
|
||||||
env.backends.onnx.wasm.numThreads = 1;
|
|
||||||
env.allowLocalModels = false;
|
|
||||||
|
|
||||||
const fileDoneBytes = new Map<string, number>();
|
|
||||||
let totalDone = 0;
|
|
||||||
let totalEstimate = 0;
|
|
||||||
|
|
||||||
this.transcriber = await pipeline('automatic-speech-recognition', modelId, {
|
|
||||||
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
|
|
||||||
if (info.status === 'progress' && info.file) {
|
|
||||||
const prevDone = fileDoneBytes.get(info.file) ?? 0;
|
|
||||||
const currentDone = info.loaded ?? 0;
|
|
||||||
const delta = Math.max(0, currentDone - prevDone);
|
|
||||||
fileDoneBytes.set(info.file, currentDone);
|
|
||||||
totalDone += delta;
|
|
||||||
if (info.total && info.total > totalEstimate) totalEstimate = info.total;
|
|
||||||
const effectiveTotal = Math.max(totalEstimate, totalDone);
|
|
||||||
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
|
|
||||||
this.setModelStatus({ state: 'downloading', progress: pct });
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
this.setModelStatus({ state: 'ready' });
|
|
||||||
} catch (err) {
|
|
||||||
const msg = err instanceof Error ? err.message : 'Unknown error loading model';
|
|
||||||
this.setModelStatus({ state: 'error', error: msg });
|
|
||||||
this.transcriber = null;
|
|
||||||
this.currentModelId = null;
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _terminateWorker(): void {
|
|
||||||
if (this.worker) {
|
|
||||||
this.worker.terminate();
|
|
||||||
this.worker = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async unloadModel(): Promise<void> {
|
|
||||||
this._terminateWorker();
|
|
||||||
this.transcriber = null;
|
|
||||||
this.currentModelId = null;
|
|
||||||
this.setModelStatus({ state: 'unloaded' });
|
|
||||||
}
|
|
||||||
|
|
||||||
async startListening(
|
|
||||||
lang: string,
|
|
||||||
onResult: SpeechResultCallback,
|
|
||||||
onError?: ErrorCallback,
|
|
||||||
): Promise<void> {
|
|
||||||
if (this.isActive) {
|
|
||||||
this.stopListening();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!this.transcriber && !this.worker) {
|
|
||||||
onError?.('Whisper model not loaded. Select a model in Voice Settings first.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.lang = lang;
|
|
||||||
this.onResult = onResult;
|
|
||||||
this.onError = onError ?? null;
|
|
||||||
this.isActive = true;
|
|
||||||
|
|
||||||
try {
|
|
||||||
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
|
||||||
} catch (err) {
|
|
||||||
this.isActive = false;
|
|
||||||
const msg = err instanceof Error ? err.message : 'Microphone access denied';
|
|
||||||
onError?.(msg);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this._setupAudioContext();
|
|
||||||
this._startRecorder();
|
|
||||||
this._startVAD();
|
|
||||||
}
|
|
||||||
|
|
||||||
stopListening(): void {
|
|
||||||
this._stopVAD();
|
|
||||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
|
||||||
try { this.mediaRecorder.stop(); } catch { /* ignore */ }
|
|
||||||
}
|
|
||||||
this._cleanupAfterStop(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
async finishListening(): Promise<void> {
|
|
||||||
if (!this.isActive) return;
|
|
||||||
|
|
||||||
this._stopVAD();
|
|
||||||
this.isSpeaking = false;
|
|
||||||
this.silenceSince = null;
|
|
||||||
|
|
||||||
if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') {
|
|
||||||
this._cleanupAfterStop(true);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await new Promise<void>((resolve) => {
|
|
||||||
this.finishResolver = resolve;
|
|
||||||
this._finaliseUtterance(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
this._cleanupAfterStop(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
getIsListening(): boolean {
|
|
||||||
return this.isActive;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Audio capture ────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
private _setupAudioContext(): void {
|
|
||||||
if (!this.stream) return;
|
|
||||||
const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
|
|
||||||
this.audioContext = new AudioContextClass();
|
|
||||||
const source = this.audioContext.createMediaStreamSource(this.stream);
|
|
||||||
this.analyser = this.audioContext.createAnalyser();
|
|
||||||
this.analyser.fftSize = 512;
|
|
||||||
source.connect(this.analyser);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _teardownAudioContext(): void {
|
|
||||||
try { this.audioContext?.close(); } catch { /* ignore */ }
|
|
||||||
this.audioContext = null;
|
|
||||||
this.analyser = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private _startRecorder(): void {
|
|
||||||
if (!this.stream) return;
|
|
||||||
const mimeType = this._pickMimeType();
|
|
||||||
const options: MediaRecorderOptions = {};
|
|
||||||
if (mimeType && MediaRecorder.isTypeSupported(mimeType)) {
|
|
||||||
options.mimeType = mimeType;
|
|
||||||
}
|
|
||||||
this.mediaRecorder = new MediaRecorder(this.stream, options);
|
|
||||||
this.chunks = [];
|
|
||||||
this.recordingStartMs = Date.now();
|
|
||||||
|
|
||||||
this.mediaRecorder.ondataavailable = (e) => {
|
|
||||||
if (e.data && e.data.size > 0) {
|
|
||||||
this.chunks.push(e.data);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
this.mediaRecorder.onstop = () => {
|
|
||||||
const blobs = this.chunks.splice(0);
|
|
||||||
const durationMs = Date.now() - this.recordingStartMs;
|
|
||||||
if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) {
|
|
||||||
this.finishResolver?.();
|
|
||||||
this.finishResolver = null;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const mType = blobs[0].type || mimeType || 'audio/webm';
|
|
||||||
const blob = new Blob(blobs, { type: mType });
|
|
||||||
void this._transcribe(blob).finally(() => {
|
|
||||||
this.finishResolver?.();
|
|
||||||
this.finishResolver = null;
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
this.mediaRecorder.start(250);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _releaseStream(): void {
|
|
||||||
if (this.stream) {
|
|
||||||
this.stream.getTracks().forEach((t) => t.stop());
|
|
||||||
this.stream = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── VAD ──────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
private _startVAD(): void {
|
|
||||||
this._stopVAD();
|
|
||||||
this.silenceSince = null;
|
|
||||||
this.isSpeaking = false;
|
|
||||||
|
|
||||||
this.vadTimer = setInterval(() => {
|
|
||||||
if (!this.isActive || !this.analyser) return;
|
|
||||||
const db = this._getRmsDb();
|
|
||||||
const isSilent = db < this.cfg.silenceThresholdDb;
|
|
||||||
|
|
||||||
if (!isSilent) {
|
|
||||||
this.silenceSince = null;
|
|
||||||
if (!this.isSpeaking) {
|
|
||||||
this.isSpeaking = true;
|
|
||||||
if (this.mediaRecorder?.state === 'recording') {
|
|
||||||
this.recordingStartMs = Date.now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (this.isSpeaking) {
|
|
||||||
if (this.silenceSince === null) {
|
|
||||||
this.silenceSince = Date.now();
|
|
||||||
} else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) {
|
|
||||||
this.isSpeaking = false;
|
|
||||||
this.silenceSince = null;
|
|
||||||
this._finaliseUtterance(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, VAD_POLL_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _stopVAD(): void {
|
|
||||||
if (this.vadTimer !== null) {
|
|
||||||
clearInterval(this.vadTimer);
|
|
||||||
this.vadTimer = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _cleanupAfterStop(clearChunks: boolean): void {
|
|
||||||
const pendingResolver = this.finishResolver;
|
|
||||||
this.isActive = false;
|
|
||||||
this.finishResolver = null;
|
|
||||||
this.mediaRecorder = null;
|
|
||||||
this._teardownAudioContext();
|
|
||||||
this._releaseStream();
|
|
||||||
if (clearChunks) this.chunks = [];
|
|
||||||
this.isSpeaking = false;
|
|
||||||
this.silenceSince = null;
|
|
||||||
this.onResult = null;
|
|
||||||
this.onError = null;
|
|
||||||
pendingResolver?.();
|
|
||||||
}
|
|
||||||
|
|
||||||
private _finaliseUtterance(restart: boolean): void {
|
|
||||||
if (!this.isActive) return;
|
|
||||||
if (this.mediaRecorder && this.mediaRecorder.state === 'recording') {
|
|
||||||
this.mediaRecorder.stop();
|
|
||||||
}
|
|
||||||
if (!restart) return;
|
|
||||||
setTimeout(() => {
|
|
||||||
if (this.isActive && this.stream) {
|
|
||||||
this._startRecorder();
|
|
||||||
}
|
|
||||||
}, 100);
|
|
||||||
}
|
|
||||||
|
|
||||||
private _getRmsDb(): number {
|
|
||||||
if (!this.analyser) return -Infinity;
|
|
||||||
const buf = new Float32Array(this.analyser.fftSize);
|
|
||||||
this.analyser.getFloatTimeDomainData(buf);
|
|
||||||
let sumSq = 0;
|
|
||||||
for (const s of buf) sumSq += s * s;
|
|
||||||
const rms = Math.sqrt(sumSq / buf.length);
|
|
||||||
return rms === 0 ? -Infinity : 20 * Math.log10(rms);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Transcription ────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
private async _transcribe(blob: Blob): Promise<void> {
|
|
||||||
if (!this.onResult) return;
|
|
||||||
if (!this.transcriber && !this.worker) {
|
|
||||||
this.onError?.('Model not loaded');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const audioData = await this._decodeToFloat32(blob);
|
|
||||||
if (!audioData || audioData.length === 0) {
|
|
||||||
this.onError?.(`Failed to decode audio (${blob.size} bytes)`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const langHint = this._resolveLanguageHint();
|
|
||||||
|
|
||||||
// Prefer worker (non-blocking); fall back to main-thread pipeline.
|
|
||||||
const transcript = this.worker
|
|
||||||
? await this._transcribeViaWorker(audioData, langHint)
|
|
||||||
: await this._transcribeMainThread(audioData, langHint);
|
|
||||||
|
|
||||||
if (transcript) {
|
|
||||||
this.onResult(transcript, true);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
if (!this.isActive) return;
|
|
||||||
const msg = err instanceof Error ? err.message : 'Local transcription failed';
|
|
||||||
this.onError?.(msg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _transcribeViaWorker(audioData: Float32Array, langHint: string | undefined): Promise<string> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
if (!this.worker) return reject(new Error('Worker gone'));
|
|
||||||
|
|
||||||
const onMessage = (e: MessageEvent) => {
|
|
||||||
const data = e.data as { type: string; error?: string; transcript?: string; text?: string };
|
|
||||||
if (data.type === 'result') {
|
|
||||||
this.worker!.removeEventListener('message', onMessage);
|
|
||||||
resolve(data.transcript ?? '');
|
|
||||||
} else if (data.type === 'log') {
|
|
||||||
console.log('[WasmStt Worker]', data.text);
|
|
||||||
} else if (data.type === 'error') {
|
|
||||||
this.worker!.removeEventListener('message', onMessage);
|
|
||||||
reject(new Error(data.error ?? 'Transcription failed'));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
this.worker.addEventListener('message', onMessage);
|
|
||||||
this.worker.postMessage(
|
|
||||||
{ type: 'transcribe', audio: audioData.buffer, language: langHint },
|
|
||||||
[audioData.buffer],
|
|
||||||
);
|
|
||||||
|
|
||||||
setTimeout(() => {
|
|
||||||
this.worker?.removeEventListener('message', onMessage);
|
|
||||||
reject(new Error('Transcription timed out'));
|
|
||||||
}, 30000);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _transcribeMainThread(audioData: Float32Array, langHint: string | undefined): Promise<string> {
|
|
||||||
const pipelineFn = this.transcriber as (
|
|
||||||
input: Float32Array,
|
|
||||||
options?: Record<string, unknown>,
|
|
||||||
) => Promise<{ text: string }>;
|
|
||||||
|
|
||||||
const result = await pipelineFn(audioData, {
|
|
||||||
task: 'transcribe',
|
|
||||||
...(langHint ? { language: langHint } : {}),
|
|
||||||
});
|
|
||||||
|
|
||||||
return (result?.text ?? '').trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _decodeToFloat32(blob: Blob): Promise<Float32Array | null> {
|
|
||||||
if (!this.audioContext) return null;
|
|
||||||
const arrayBuffer = await blob.arrayBuffer();
|
|
||||||
let audioBuffer: AudioBuffer;
|
|
||||||
try {
|
|
||||||
audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const origRate = audioBuffer.sampleRate;
|
|
||||||
const origData = audioBuffer.getChannelData(0);
|
|
||||||
const targetRate = WHISPER_SAMPLE_RATE;
|
|
||||||
|
|
||||||
if (origRate === targetRate) {
|
|
||||||
return new Float32Array(origData);
|
|
||||||
}
|
|
||||||
|
|
||||||
const ratio = origRate / targetRate;
|
|
||||||
const newLength = Math.ceil(origData.length / ratio);
|
|
||||||
const result = new Float32Array(newLength);
|
|
||||||
for (let i = 0; i < newLength; i++) {
|
|
||||||
const origIdx = i * ratio;
|
|
||||||
const idx0 = Math.floor(origIdx);
|
|
||||||
const idx1 = Math.min(idx0 + 1, origData.length - 1);
|
|
||||||
const frac = origIdx - idx0;
|
|
||||||
result[i] = origData[idx0] * (1 - frac) + origData[idx1] * frac;
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private _resolveLanguageHint(): string | undefined {
|
|
||||||
if (this.lang && this.lang !== 'auto') {
|
|
||||||
return this.lang.split('-')[0];
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
private _pickMimeType(): string {
|
|
||||||
const candidates = [
|
|
||||||
'audio/webm;codecs=opus',
|
|
||||||
'audio/webm',
|
|
||||||
'audio/ogg;codecs=opus',
|
|
||||||
'audio/ogg',
|
|
||||||
'audio/mp4',
|
|
||||||
];
|
|
||||||
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) {
|
|
||||||
return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? '';
|
|
||||||
}
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const wasmSttService = new WasmSttService();
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
/**
|
|
||||||
* Web Worker for off-main-thread Whisper transcription.
|
|
||||||
*
|
|
||||||
* Receives `{ type: 'load', modelId }` to load a model, then
|
|
||||||
* `{ type: 'transcribe', audio: Float32Array (transferred buffer), language? }`
|
|
||||||
* to run inference. Posts progress, results, and errors back.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { pipeline, env } from '@xenova/transformers';
|
|
||||||
|
|
||||||
let transcriber: unknown = null;
|
|
||||||
|
|
||||||
self.onmessage = async (e: MessageEvent) => {
|
|
||||||
const { type } = e.data as { type: string };
|
|
||||||
|
|
||||||
if (type === 'load') {
|
|
||||||
const { modelId } = e.data as { modelId: string };
|
|
||||||
try {
|
|
||||||
env.backends.onnx.wasm.numThreads = 1;
|
|
||||||
|
|
||||||
const fileDoneBytes = new Map<string, number>();
|
|
||||||
let totalDone = 0;
|
|
||||||
let totalEstimate = 0;
|
|
||||||
|
|
||||||
transcriber = await pipeline('automatic-speech-recognition', modelId, {
|
|
||||||
progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => {
|
|
||||||
if (info.status === 'progress' && info.file) {
|
|
||||||
const prevDone = fileDoneBytes.get(info.file) ?? 0;
|
|
||||||
const currentDone = info.loaded ?? 0;
|
|
||||||
const delta = Math.max(0, currentDone - prevDone);
|
|
||||||
fileDoneBytes.set(info.file, currentDone);
|
|
||||||
totalDone += delta;
|
|
||||||
|
|
||||||
if (info.total && info.total > totalEstimate) {
|
|
||||||
totalEstimate = info.total;
|
|
||||||
}
|
|
||||||
|
|
||||||
const effectiveTotal = Math.max(totalEstimate, totalDone);
|
|
||||||
const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0;
|
|
||||||
self.postMessage({ type: 'progress', progress: pct });
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
self.postMessage({ type: 'loaded' });
|
|
||||||
} catch (err) {
|
|
||||||
self.postMessage({
|
|
||||||
type: 'error',
|
|
||||||
error: err instanceof Error ? err.message : 'Failed to load model',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else if (type === 'transcribe') {
|
|
||||||
if (!transcriber) {
|
|
||||||
self.postMessage({ type: 'error', error: 'Model not loaded', seq: (e.data as { seq?: number }).seq });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { audio, language, seq } = e.data as { audio: ArrayBuffer; language?: string; seq?: number };
|
|
||||||
|
|
||||||
try {
|
|
||||||
const samples = new Float32Array(audio);
|
|
||||||
if (samples.length === 0) {
|
|
||||||
self.postMessage({ type: 'error', error: 'Empty audio received', seq });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
self.postMessage({ type: 'log', text: `Transcribing ${samples.length} samples (${(samples.length / 16000).toFixed(1)}s)` });
|
|
||||||
|
|
||||||
const pipelineFn = transcriber as (
|
|
||||||
input: Float32Array,
|
|
||||||
options?: Record<string, unknown>,
|
|
||||||
) => Promise<{ text: string }>;
|
|
||||||
|
|
||||||
const result = await pipelineFn(samples, {
|
|
||||||
task: 'transcribe',
|
|
||||||
...(language ? { language } : {}),
|
|
||||||
});
|
|
||||||
|
|
||||||
self.postMessage({
|
|
||||||
type: 'result',
|
|
||||||
transcript: (result?.text ?? '').trim(),
|
|
||||||
seq,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
self.postMessage({
|
|
||||||
type: 'error',
|
|
||||||
error: err instanceof Error ? err.message : 'Transcription failed',
|
|
||||||
seq,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -23,10 +23,6 @@ import { getSyncConfig, subscribeToSyncConfigChanges } from "@/sync/sync-refs";
|
|||||||
|
|
||||||
const MODELS_DEV_API_URL = "https://models.dev/api.json";
|
const MODELS_DEV_API_URL = "https://models.dev/api.json";
|
||||||
const MODELS_DEV_PROXY_URL = "/api/openchamber/models-metadata";
|
const MODELS_DEV_PROXY_URL = "/api/openchamber/models-metadata";
|
||||||
const STT_SILENCE_THRESHOLD_DB_MIN = -100;
|
|
||||||
const STT_SILENCE_THRESHOLD_DB_MAX = 0;
|
|
||||||
const STT_SILENCE_HOLD_MS_MIN = 250;
|
|
||||||
const STT_SILENCE_HOLD_MS_MAX = 10000;
|
|
||||||
|
|
||||||
const FALLBACK_PROVIDER_ID = "opencode";
|
const FALLBACK_PROVIDER_ID = "opencode";
|
||||||
const FALLBACK_MODEL_ID = "big-pickle";
|
const FALLBACK_MODEL_ID = "big-pickle";
|
||||||
@@ -38,18 +34,19 @@ const GIT_UTILITY_PROVIDER_ID = "zen";
|
|||||||
const GIT_UTILITY_PREFERRED_MODEL_ID = "big-pickle";
|
const GIT_UTILITY_PREFERRED_MODEL_ID = "big-pickle";
|
||||||
const PROVIDER_CONFIG_REFRESH_CONCURRENCY = 4;
|
const PROVIDER_CONFIG_REFRESH_CONCURRENCY = 4;
|
||||||
|
|
||||||
const normalizeSttSilenceThresholdDb = (value: unknown): number | undefined => {
|
const normalizeSttProvider = (value: unknown): 'local' | 'openai-compatible' | undefined => {
|
||||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
if (value === 'local' || value === 'openai-compatible') {
|
||||||
return undefined;
|
return value;
|
||||||
}
|
}
|
||||||
return Math.max(STT_SILENCE_THRESHOLD_DB_MIN, Math.min(STT_SILENCE_THRESHOLD_DB_MAX, value));
|
// Legacy providers: 'server' used an OpenAI-compatible endpoint;
|
||||||
};
|
// 'browser' and 'wasm' map to the local default.
|
||||||
|
if (value === 'server') {
|
||||||
const normalizeSttSilenceHoldMs = (value: unknown): number | undefined => {
|
return 'openai-compatible';
|
||||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
||||||
return undefined;
|
|
||||||
}
|
}
|
||||||
return Math.max(STT_SILENCE_HOLD_MS_MIN, Math.min(STT_SILENCE_HOLD_MS_MAX, Math.round(value)));
|
if (value === 'browser' || value === 'wasm') {
|
||||||
|
return 'local';
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
};
|
};
|
||||||
|
|
||||||
interface OpenChamberDefaults {
|
interface OpenChamberDefaults {
|
||||||
@@ -61,13 +58,11 @@ interface OpenChamberDefaults {
|
|||||||
defaultFileViewerPreview?: boolean;
|
defaultFileViewerPreview?: boolean;
|
||||||
zenModel?: string;
|
zenModel?: string;
|
||||||
messageStreamTransport?: 'auto' | 'ws' | 'sse';
|
messageStreamTransport?: 'auto' | 'ws' | 'sse';
|
||||||
sttProvider?: 'browser' | 'server' | 'wasm';
|
sttProvider?: 'local' | 'openai-compatible';
|
||||||
sttServerUrl?: string;
|
sttServerUrl?: string;
|
||||||
wasmSttModel?: string;
|
|
||||||
sttModel?: string;
|
sttModel?: string;
|
||||||
|
sttLocalModel?: string;
|
||||||
sttLanguage?: string;
|
sttLanguage?: string;
|
||||||
sttSilenceThresholdDb?: number;
|
|
||||||
sttSilenceHoldMs?: number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
||||||
@@ -101,12 +96,11 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
|||||||
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
|
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
|
||||||
? data.messageStreamTransport
|
? data.messageStreamTransport
|
||||||
: undefined;
|
: undefined;
|
||||||
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' || data?.sttProvider === 'wasm' ? data.sttProvider : undefined;
|
const sttProvider = normalizeSttProvider(data?.sttProvider);
|
||||||
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
|
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
|
||||||
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
|
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
|
||||||
|
const sttLocalModel = typeof data?.sttLocalModel === 'string' ? data.sttLocalModel.trim() : undefined;
|
||||||
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
|
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
|
||||||
const sttSilenceThresholdDb = normalizeSttSilenceThresholdDb(data?.sttSilenceThresholdDb);
|
|
||||||
const sttSilenceHoldMs = normalizeSttSilenceHoldMs(data?.sttSilenceHoldMs);
|
|
||||||
|
|
||||||
return finish('runtime-settings', {
|
return finish('runtime-settings', {
|
||||||
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
|
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
|
||||||
@@ -120,9 +114,8 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
|||||||
sttProvider,
|
sttProvider,
|
||||||
sttServerUrl,
|
sttServerUrl,
|
||||||
sttModel,
|
sttModel,
|
||||||
|
sttLocalModel,
|
||||||
sttLanguage,
|
sttLanguage,
|
||||||
sttSilenceThresholdDb,
|
|
||||||
sttSilenceHoldMs,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
@@ -149,12 +142,11 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
|||||||
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
|
data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto'
|
||||||
? data.messageStreamTransport
|
? data.messageStreamTransport
|
||||||
: undefined;
|
: undefined;
|
||||||
const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' ? data.sttProvider : undefined;
|
const sttProvider = normalizeSttProvider(data?.sttProvider);
|
||||||
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
|
const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined;
|
||||||
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
|
const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined;
|
||||||
|
const sttLocalModel = typeof data?.sttLocalModel === 'string' ? data.sttLocalModel.trim() : undefined;
|
||||||
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
|
const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined;
|
||||||
const sttSilenceThresholdDb = normalizeSttSilenceThresholdDb(data?.sttSilenceThresholdDb);
|
|
||||||
const sttSilenceHoldMs = normalizeSttSilenceHoldMs(data?.sttSilenceHoldMs);
|
|
||||||
|
|
||||||
return finish('settings-route', {
|
return finish('settings-route', {
|
||||||
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
|
defaultModel: defaultModel.length > 0 ? defaultModel : undefined,
|
||||||
@@ -168,9 +160,8 @@ const fetchOpenChamberDefaults = async (): Promise<OpenChamberDefaults> => {
|
|||||||
sttProvider,
|
sttProvider,
|
||||||
sttServerUrl,
|
sttServerUrl,
|
||||||
sttModel,
|
sttModel,
|
||||||
|
sttLocalModel,
|
||||||
sttLanguage,
|
sttLanguage,
|
||||||
sttSilenceThresholdDb,
|
|
||||||
sttSilenceHoldMs,
|
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
markStartupTrace('config.defaults:error', { error: error instanceof Error ? error.message : String(error) });
|
markStartupTrace('config.defaults:error', { error: error instanceof Error ? error.message : String(error) });
|
||||||
@@ -989,33 +980,31 @@ interface ConfigStore {
|
|||||||
settingsZenModel: string | undefined;
|
settingsZenModel: string | undefined;
|
||||||
settingsMessageStreamTransport: 'auto' | 'ws' | 'sse';
|
settingsMessageStreamTransport: 'auto' | 'ws' | 'sse';
|
||||||
// Voice provider preference ('browser', 'openai', 'openai-compatible', or 'say' for macOS)
|
// Voice provider preference ('browser', 'openai', 'openai-compatible', or 'say' for macOS)
|
||||||
voiceProvider: 'browser' | 'openai' | 'openai-compatible' | 'say';
|
voiceProvider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say';
|
||||||
setVoiceProvider: (provider: 'browser' | 'openai' | 'openai-compatible' | 'say') => void;
|
setVoiceProvider: (provider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say') => void;
|
||||||
// TTS settings
|
// TTS settings
|
||||||
speechRate: number;
|
speechRate: number;
|
||||||
speechPitch: number;
|
speechPitch: number;
|
||||||
speechVolume: number;
|
speechVolume: number;
|
||||||
sayVoice: string;
|
sayVoice: string;
|
||||||
browserVoice: string;
|
browserVoice: string;
|
||||||
|
localTtsVoiceId: number;
|
||||||
openaiVoice: string;
|
openaiVoice: string;
|
||||||
openaiApiKey: string;
|
openaiApiKey: string;
|
||||||
openaiCompatibleUrl: string;
|
openaiCompatibleUrl: string;
|
||||||
openaiCompatibleApiKey: string;
|
openaiCompatibleApiKey: string;
|
||||||
openaiCompatibleVoice: string;
|
openaiCompatibleVoice: string;
|
||||||
openaiCompatibleTtsModel: string;
|
openaiCompatibleTtsModel: string;
|
||||||
// STT (speech-to-text) settings
|
// STT (dictation) settings
|
||||||
sttProvider: 'browser' | 'server' | 'wasm';
|
dictationEnabled: boolean;
|
||||||
|
sttProvider: 'local' | 'openai-compatible';
|
||||||
sttServerUrl: string;
|
sttServerUrl: string;
|
||||||
sttApiKey: string;
|
sttApiKey: string;
|
||||||
sttModel: string;
|
sttModel: string;
|
||||||
wasmSttModel: string;
|
sttLocalModel: string;
|
||||||
sttLanguage: string;
|
sttLanguage: string;
|
||||||
sttSilenceThresholdDb: number;
|
|
||||||
sttSilenceHoldMs: number;
|
|
||||||
sttTranscribeOnStop: boolean;
|
|
||||||
showMessageTTSButtons: boolean;
|
showMessageTTSButtons: boolean;
|
||||||
ttsInputMode: 'sanitized' | 'raw';
|
ttsInputMode: 'sanitized' | 'raw';
|
||||||
voiceModeEnabled: boolean;
|
|
||||||
// Summarization settings
|
// Summarization settings
|
||||||
summarizeMessageTTS: boolean;
|
summarizeMessageTTS: boolean;
|
||||||
summarizeVoiceConversation: boolean;
|
summarizeVoiceConversation: boolean;
|
||||||
@@ -1026,24 +1015,22 @@ interface ConfigStore {
|
|||||||
setSpeechVolume: (volume: number) => void;
|
setSpeechVolume: (volume: number) => void;
|
||||||
setSayVoice: (voice: string) => void;
|
setSayVoice: (voice: string) => void;
|
||||||
setBrowserVoice: (voice: string) => void;
|
setBrowserVoice: (voice: string) => void;
|
||||||
|
setLocalTtsVoiceId: (voiceId: number) => void;
|
||||||
setOpenaiVoice: (voice: string) => void;
|
setOpenaiVoice: (voice: string) => void;
|
||||||
setOpenaiApiKey: (apiKey: string) => void;
|
setOpenaiApiKey: (apiKey: string) => void;
|
||||||
setOpenaiCompatibleUrl: (url: string) => void;
|
setOpenaiCompatibleUrl: (url: string) => void;
|
||||||
setOpenaiCompatibleApiKey: (apiKey: string) => void;
|
setOpenaiCompatibleApiKey: (apiKey: string) => void;
|
||||||
setOpenaiCompatibleVoice: (voice: string) => void;
|
setOpenaiCompatibleVoice: (voice: string) => void;
|
||||||
setOpenaiCompatibleTtsModel: (model: string) => void;
|
setOpenaiCompatibleTtsModel: (model: string) => void;
|
||||||
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => void;
|
setDictationEnabled: (enabled: boolean) => void;
|
||||||
|
setSttProvider: (provider: 'local' | 'openai-compatible') => void;
|
||||||
setSttServerUrl: (url: string) => void;
|
setSttServerUrl: (url: string) => void;
|
||||||
setSttApiKey: (apiKey: string) => void;
|
setSttApiKey: (apiKey: string) => void;
|
||||||
setSttModel: (model: string) => void;
|
setSttModel: (model: string) => void;
|
||||||
setWasmSttModel: (model: string) => void;
|
setSttLocalModel: (model: string) => void;
|
||||||
setSttLanguage: (lang: string) => void;
|
setSttLanguage: (lang: string) => void;
|
||||||
setSttSilenceThresholdDb: (db: number) => void;
|
|
||||||
setSttSilenceHoldMs: (ms: number) => void;
|
|
||||||
setSttTranscribeOnStop: (enabled: boolean) => void;
|
|
||||||
setShowMessageTTSButtons: (show: boolean) => void;
|
setShowMessageTTSButtons: (show: boolean) => void;
|
||||||
setTtsInputMode: (mode: 'sanitized' | 'raw') => void;
|
setTtsInputMode: (mode: 'sanitized' | 'raw') => void;
|
||||||
setVoiceModeEnabled: (enabled: boolean) => void;
|
|
||||||
setSummarizeMessageTTS: (enabled: boolean) => void;
|
setSummarizeMessageTTS: (enabled: boolean) => void;
|
||||||
setSummarizeVoiceConversation: (enabled: boolean) => void;
|
setSummarizeVoiceConversation: (enabled: boolean) => void;
|
||||||
setSummarizeCharacterThreshold: (threshold: number) => void;
|
setSummarizeCharacterThreshold: (threshold: number) => void;
|
||||||
@@ -1136,7 +1123,7 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
voiceProvider: (() => {
|
voiceProvider: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
const saved = localStorage.getItem('voiceProvider');
|
const saved = localStorage.getItem('voiceProvider');
|
||||||
if (saved === 'openai' || saved === 'browser' || saved === 'say' || saved === 'openai-compatible') return saved;
|
if (saved === 'openai' || saved === 'browser' || saved === 'local' || saved === 'say' || saved === 'openai-compatible') return saved;
|
||||||
}
|
}
|
||||||
return 'browser';
|
return 'browser';
|
||||||
})(),
|
})(),
|
||||||
@@ -1179,6 +1166,17 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
return 'Samantha';
|
return 'Samantha';
|
||||||
})(),
|
})(),
|
||||||
|
// Local (Kokoro) TTS speaker id - load from localStorage or default to 0
|
||||||
|
localTtsVoiceId: (() => {
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
const saved = localStorage.getItem('localTtsVoiceId');
|
||||||
|
if (saved !== null) {
|
||||||
|
const parsed = Number.parseInt(saved, 10);
|
||||||
|
if (Number.isInteger(parsed) && parsed >= 0) return parsed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
})(),
|
||||||
// Browser voice - load from localStorage or default to empty (auto-select)
|
// Browser voice - load from localStorage or default to empty (auto-select)
|
||||||
browserVoice: (() => {
|
browserVoice: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -1235,17 +1233,24 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
return 'kokoro';
|
return 'kokoro';
|
||||||
})(),
|
})(),
|
||||||
// STT provider: 'browser' (Web Speech API), 'server' (OpenAI-compat), 'wasm' (local Whisper)
|
// Voice input (dictation) master toggle - default enabled
|
||||||
|
dictationEnabled: (() => {
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
const saved = localStorage.getItem('dictationEnabled');
|
||||||
|
if (saved === 'false') return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
})(),
|
||||||
|
// STT provider: 'local' (server-side sherpa-onnx) or 'openai-compatible'
|
||||||
sttProvider: (() => {
|
sttProvider: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
const saved = localStorage.getItem('sttProvider');
|
const saved = localStorage.getItem('sttProvider');
|
||||||
if (saved === 'browser' || saved === 'server' || saved === 'wasm') return saved;
|
if (saved === 'local' || saved === 'openai-compatible') return saved;
|
||||||
// Electron/Chromium's Web Speech API requires Google API keys
|
// Migrate legacy providers: 'server' used an OpenAI-compatible
|
||||||
// not available in Electron, so default to WASM local Whisper.
|
// endpoint; 'browser' and 'wasm' map to the local default.
|
||||||
const electron = (window as unknown as { __OPENCHAMBER_ELECTRON__?: { runtime?: string } }).__OPENCHAMBER_ELECTRON__;
|
if (saved === 'server') return 'openai-compatible' as const;
|
||||||
if (electron?.runtime === 'electron') return 'wasm' as const;
|
|
||||||
}
|
}
|
||||||
return 'browser' as const;
|
return 'local' as const;
|
||||||
})(),
|
})(),
|
||||||
sttServerUrl: (() => {
|
sttServerUrl: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -1268,12 +1273,12 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
return 'deepdml/faster-whisper-large-v3-turbo-ct2';
|
return 'deepdml/faster-whisper-large-v3-turbo-ct2';
|
||||||
})(),
|
})(),
|
||||||
wasmSttModel: (() => {
|
sttLocalModel: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
const saved = localStorage.getItem('wasmSttModel');
|
const saved = localStorage.getItem('sttLocalModel');
|
||||||
if (saved) return saved;
|
if (saved) return saved;
|
||||||
}
|
}
|
||||||
return 'Xenova/whisper-base.en';
|
return 'parakeet-tdt-0.6b-v2-int8';
|
||||||
})(),
|
})(),
|
||||||
sttLanguage: (() => {
|
sttLanguage: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -1282,33 +1287,6 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
return '';
|
return '';
|
||||||
})(),
|
})(),
|
||||||
sttSilenceThresholdDb: (() => {
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
const saved = localStorage.getItem('sttSilenceThresholdDb');
|
|
||||||
if (saved) {
|
|
||||||
const parsed = parseFloat(saved);
|
|
||||||
if (!isNaN(parsed)) return parsed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -45;
|
|
||||||
})(),
|
|
||||||
sttSilenceHoldMs: (() => {
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
const saved = localStorage.getItem('sttSilenceHoldMs');
|
|
||||||
if (saved) {
|
|
||||||
const parsed = parseInt(saved, 10);
|
|
||||||
if (!isNaN(parsed)) return parsed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 1500;
|
|
||||||
})(),
|
|
||||||
sttTranscribeOnStop: (() => {
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
const saved = localStorage.getItem('sttTranscribeOnStop');
|
|
||||||
if (saved === 'true') return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
})(),
|
|
||||||
// Show TTS buttons on messages - disabled by default until user enables it
|
// Show TTS buttons on messages - disabled by default until user enables it
|
||||||
showMessageTTSButtons: (() => {
|
showMessageTTSButtons: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -1324,14 +1302,6 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
return 'sanitized' as const;
|
return 'sanitized' as const;
|
||||||
})(),
|
})(),
|
||||||
// Voice mode enabled - load from localStorage or default to false
|
|
||||||
voiceModeEnabled: (() => {
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
const saved = localStorage.getItem('voiceModeEnabled');
|
|
||||||
if (saved === 'true') return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
})(),
|
|
||||||
// Summarization settings
|
// Summarization settings
|
||||||
summarizeMessageTTS: (() => {
|
summarizeMessageTTS: (() => {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -2060,9 +2030,8 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
sttProvider: openChamberDefaults.sttProvider ?? state.sttProvider,
|
sttProvider: openChamberDefaults.sttProvider ?? state.sttProvider,
|
||||||
sttServerUrl: openChamberDefaults.sttServerUrl ?? state.sttServerUrl,
|
sttServerUrl: openChamberDefaults.sttServerUrl ?? state.sttServerUrl,
|
||||||
sttModel: openChamberDefaults.sttModel ?? state.sttModel,
|
sttModel: openChamberDefaults.sttModel ?? state.sttModel,
|
||||||
|
sttLocalModel: openChamberDefaults.sttLocalModel ?? state.sttLocalModel,
|
||||||
sttLanguage: openChamberDefaults.sttLanguage ?? state.sttLanguage,
|
sttLanguage: openChamberDefaults.sttLanguage ?? state.sttLanguage,
|
||||||
sttSilenceThresholdDb: openChamberDefaults.sttSilenceThresholdDb ?? state.sttSilenceThresholdDb,
|
|
||||||
sttSilenceHoldMs: openChamberDefaults.sttSilenceHoldMs ?? state.sttSilenceHoldMs,
|
|
||||||
directoryScoped: {
|
directoryScoped: {
|
||||||
...state.directoryScoped,
|
...state.directoryScoped,
|
||||||
[directoryKey]: nextSnapshot,
|
[directoryKey]: nextSnapshot,
|
||||||
@@ -2800,7 +2769,7 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
setVoiceProvider: (provider: 'browser' | 'openai' | 'openai-compatible' | 'say') => {
|
setVoiceProvider: (provider: 'browser' | 'local' | 'openai' | 'openai-compatible' | 'say') => {
|
||||||
set({ voiceProvider: provider });
|
set({ voiceProvider: provider });
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
localStorage.setItem('voiceProvider', provider);
|
localStorage.setItem('voiceProvider', provider);
|
||||||
@@ -2838,6 +2807,13 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
setLocalTtsVoiceId: (voiceId: number) => {
|
||||||
|
set({ localTtsVoiceId: voiceId });
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
localStorage.setItem('localTtsVoiceId', String(voiceId));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
setBrowserVoice: (voice: string) => {
|
setBrowserVoice: (voice: string) => {
|
||||||
set({ browserVoice: voice });
|
set({ browserVoice: voice });
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -2887,7 +2863,15 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
setSttProvider: (provider: 'browser' | 'server' | 'wasm') => {
|
setDictationEnabled: (enabled: boolean) => {
|
||||||
|
set({ dictationEnabled: enabled });
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
localStorage.setItem('dictationEnabled', String(enabled));
|
||||||
|
}
|
||||||
|
updateDesktopSettings({ dictationEnabled: enabled }).catch(() => {});
|
||||||
|
},
|
||||||
|
|
||||||
|
setSttProvider: (provider: 'local' | 'openai-compatible') => {
|
||||||
set({ sttProvider: provider });
|
set({ sttProvider: provider });
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
localStorage.setItem('sttProvider', provider);
|
localStorage.setItem('sttProvider', provider);
|
||||||
@@ -2918,12 +2902,12 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
updateDesktopSettings({ sttModel: model }).catch(() => {});
|
updateDesktopSettings({ sttModel: model }).catch(() => {});
|
||||||
},
|
},
|
||||||
|
|
||||||
setWasmSttModel: (model: string) => {
|
setSttLocalModel: (model: string) => {
|
||||||
set({ wasmSttModel: model });
|
set({ sttLocalModel: model });
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
localStorage.setItem('wasmSttModel', model);
|
localStorage.setItem('sttLocalModel', model);
|
||||||
}
|
}
|
||||||
updateDesktopSettings({ wasmSttModel: model }).catch(() => {});
|
updateDesktopSettings({ sttLocalModel: model }).catch(() => {});
|
||||||
},
|
},
|
||||||
|
|
||||||
setSttLanguage: (lang: string) => {
|
setSttLanguage: (lang: string) => {
|
||||||
@@ -2934,30 +2918,6 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
updateDesktopSettings({ sttLanguage: lang }).catch(() => {});
|
updateDesktopSettings({ sttLanguage: lang }).catch(() => {});
|
||||||
},
|
},
|
||||||
|
|
||||||
setSttSilenceThresholdDb: (db: number) => {
|
|
||||||
set({ sttSilenceThresholdDb: db });
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
localStorage.setItem('sttSilenceThresholdDb', String(db));
|
|
||||||
}
|
|
||||||
updateDesktopSettings({ sttSilenceThresholdDb: db }).catch(() => {});
|
|
||||||
},
|
|
||||||
|
|
||||||
setSttSilenceHoldMs: (ms: number) => {
|
|
||||||
set({ sttSilenceHoldMs: ms });
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
localStorage.setItem('sttSilenceHoldMs', String(ms));
|
|
||||||
}
|
|
||||||
updateDesktopSettings({ sttSilenceHoldMs: ms }).catch(() => {});
|
|
||||||
},
|
|
||||||
|
|
||||||
setSttTranscribeOnStop: (enabled: boolean) => {
|
|
||||||
set({ sttTranscribeOnStop: enabled });
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
localStorage.setItem('sttTranscribeOnStop', String(enabled));
|
|
||||||
}
|
|
||||||
updateDesktopSettings({ sttTranscribeOnStop: enabled }).catch(() => {});
|
|
||||||
},
|
|
||||||
|
|
||||||
setShowMessageTTSButtons: (show: boolean) => {
|
setShowMessageTTSButtons: (show: boolean) => {
|
||||||
set({ showMessageTTSButtons: show });
|
set({ showMessageTTSButtons: show });
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
@@ -2972,13 +2932,6 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
setVoiceModeEnabled: (enabled: boolean) => {
|
|
||||||
set({ voiceModeEnabled: enabled });
|
|
||||||
if (typeof window !== 'undefined') {
|
|
||||||
localStorage.setItem('voiceModeEnabled', String(enabled));
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
setSummarizeMessageTTS: (enabled: boolean) => {
|
setSummarizeMessageTTS: (enabled: boolean) => {
|
||||||
set({ summarizeMessageTTS: enabled });
|
set({ summarizeMessageTTS: enabled });
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
|
|||||||
@@ -190,7 +190,6 @@ function notifyMessageSent(sessionId: string): void {
|
|||||||
|
|
||||||
export type { SyntheticContextPart } from "./input-store"
|
export type { SyntheticContextPart } from "./input-store"
|
||||||
export type { SessionMemoryState } from "./viewport-store"
|
export type { SessionMemoryState } from "./viewport-store"
|
||||||
export type { VoiceStatus, VoiceMode } from "./voice-store"
|
|
||||||
|
|
||||||
export type NewSessionDraftState = {
|
export type NewSessionDraftState = {
|
||||||
open: boolean
|
open: boolean
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import type { OpencodeClient } from "@opencode-ai/sdk/v2/client"
|
|||||||
import { createEventPipeline } from "./event-pipeline"
|
import { createEventPipeline } from "./event-pipeline"
|
||||||
import { isVSCodeRuntime } from "@/lib/desktop"
|
import { isVSCodeRuntime } from "@/lib/desktop"
|
||||||
import { isMobileSurfaceRuntime } from "@/lib/runtimeSurface"
|
import { isMobileSurfaceRuntime } from "@/lib/runtimeSurface"
|
||||||
import { isCapacitorApp } from "@/lib/platform"
|
|
||||||
import { reduceGlobalEvent, applyGlobalProject, applyDirectoryEvent, type SessionMaterializationReason } from "./event-reducer"
|
import { reduceGlobalEvent, applyGlobalProject, applyDirectoryEvent, type SessionMaterializationReason } from "./event-reducer"
|
||||||
import { useGlobalSyncStore } from "./global-sync-store"
|
import { useGlobalSyncStore } from "./global-sync-store"
|
||||||
import { ChildStoreManager, type DirectoryStore } from "./child-store"
|
import { ChildStoreManager, type DirectoryStore } from "./child-store"
|
||||||
@@ -1692,12 +1691,13 @@ export function SyncProvider(props: {
|
|||||||
directory: string
|
directory: string
|
||||||
children: React.ReactNode
|
children: React.ReactNode
|
||||||
}) {
|
}) {
|
||||||
const storedMessageStreamTransport = useConfigStore((state) => state.settingsMessageStreamTransport)
|
// Capacitor apps were previously locked to SSE because Android WebSocket
|
||||||
// Capacitor apps are locked to SSE: native WebSocket streaming is unreliable there (on
|
// upgrades appeared broken. Root cause was server-side: the Android WebView
|
||||||
// Android events only arrive once the run finishes), while SSE streams correctly. The Chat
|
// origin (https://localhost, androidScheme 'https') was missing from the
|
||||||
// settings UI disables the other options on mobile, but force it here too so the effective
|
// packaged-client origin allowlist, so every WS upgrade was rejected with
|
||||||
// transport can't drift. Remove this override (and the UI lock) to re-enable WS on mobile.
|
// 403. With the origin allowlisted, mobile uses the same transport
|
||||||
const messageStreamTransport: 'auto' | 'ws' | 'sse' = isCapacitorApp() ? 'sse' : storedMessageStreamTransport
|
// selection as everywhere else ('auto' falls back to SSE on WS failure).
|
||||||
|
const messageStreamTransport = useConfigStore((state) => state.settingsMessageStreamTransport)
|
||||||
const childStoresRef = useRef<ChildStoreManager | null>(null)
|
const childStoresRef = useRef<ChildStoreManager | null>(null)
|
||||||
if (!childStoresRef.current) childStoresRef.current = new ChildStoreManager()
|
if (!childStoresRef.current) childStoresRef.current = new ChildStoreManager()
|
||||||
const childStores = childStoresRef.current
|
const childStores = childStoresRef.current
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
/**
|
|
||||||
* Voice Store — voice connection and activity state.
|
|
||||||
* Extracted from session-ui-store for subscription isolation.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export type VoiceStatus = "disconnected" | "connecting" | "connected" | "error"
|
|
||||||
export type VoiceMode = "idle" | "speaking" | "listening"
|
|
||||||
@@ -520,6 +520,23 @@ const handleLocalApiRequest = async (input: RequestInfo | URL, url: URL, init: R
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Dictation runs on the OpenChamber web server (WebSocket + worker); the VS
|
||||||
|
// Code bridge has no server process, so report it deterministically
|
||||||
|
// unavailable. The mic button hides itself when capture is unsupported.
|
||||||
|
if (normalizedPathname === '/api/dictation/status' && method === 'GET') {
|
||||||
|
return new Response(JSON.stringify({ provider: 'local', available: false, reasonCode: 'unsupported_runtime', models: [] }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedPathname.startsWith('/api/dictation/') ) {
|
||||||
|
return new Response(JSON.stringify({ error: 'Dictation is not available in VS Code runtime' }), {
|
||||||
|
status: 501,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Health endpoints: reflect actual connection status
|
// Health endpoints: reflect actual connection status
|
||||||
if (pathname === '/health' || pathname === '/api/health') {
|
if (pathname === '/health' || pathname === '/api/health') {
|
||||||
const connectionStatus = window.__OPENCHAMBER_CONNECTION__?.status;
|
const connectionStatus = window.__OPENCHAMBER_CONNECTION__?.status;
|
||||||
|
|||||||
@@ -1,2 +1,3 @@
|
|||||||
server/prompt-templates.js
|
server/prompt-templates.js
|
||||||
prompt-enhancer-config.json
|
prompt-enhancer-config.json
|
||||||
|
test-results/
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
"openai": "^4.79.0",
|
"openai": "^4.79.0",
|
||||||
"qrcode-terminal": "^0.12.0",
|
"qrcode-terminal": "^0.12.0",
|
||||||
"reflect-metadata": "^0.2.2",
|
"reflect-metadata": "^0.2.2",
|
||||||
|
"sherpa-onnx-node": "1.12.28",
|
||||||
"simple-git": "^3.28.0",
|
"simple-git": "^3.28.0",
|
||||||
"web-push": "^3.6.7",
|
"web-push": "^3.6.7",
|
||||||
"ws": "^8.18.3",
|
"ws": "^8.18.3",
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ import { prepareNotificationLastMessage } from './lib/notifications/index.js';
|
|||||||
import { registerTtsRoutes } from './lib/tts/routes.js';
|
import { registerTtsRoutes } from './lib/tts/routes.js';
|
||||||
import { detectSayTtsCapability } from './lib/tts/capability-runtime.js';
|
import { detectSayTtsCapability } from './lib/tts/capability-runtime.js';
|
||||||
import { createTerminalRuntime } from './lib/terminal/runtime.js';
|
import { createTerminalRuntime } from './lib/terminal/runtime.js';
|
||||||
|
import { createDictationRuntime } from './lib/dictation/runtime.js';
|
||||||
import {
|
import {
|
||||||
createGlobalUiEventBroadcaster,
|
createGlobalUiEventBroadcaster,
|
||||||
createGlobalMessageStreamHub,
|
createGlobalMessageStreamHub,
|
||||||
@@ -495,6 +496,7 @@ const tunnelAuthController = createTunnelAuth();
|
|||||||
let runtimeManagedRemoteTunnelToken = '';
|
let runtimeManagedRemoteTunnelToken = '';
|
||||||
let runtimeManagedRemoteTunnelHostname = '';
|
let runtimeManagedRemoteTunnelHostname = '';
|
||||||
let terminalRuntime = null;
|
let terminalRuntime = null;
|
||||||
|
let dictationRuntime = null;
|
||||||
let messageStreamRuntime = null;
|
let messageStreamRuntime = null;
|
||||||
const userProvidedOpenCodePassword = hmrStateRuntime.getUserProvidedOpenCodePassword(hmrState);
|
const userProvidedOpenCodePassword = hmrStateRuntime.getUserProvidedOpenCodePassword(hmrState);
|
||||||
const initialOpenCodeAuthState = hmrStateRuntime.resolveOpenCodeAuthFromState({
|
const initialOpenCodeAuthState = hmrStateRuntime.resolveOpenCodeAuthFromState({
|
||||||
@@ -898,6 +900,7 @@ const tunnelWiringRuntime = createTunnelWiringRuntime({
|
|||||||
});
|
});
|
||||||
const startupPipelineRuntime = createStartupPipelineRuntime({
|
const startupPipelineRuntime = createStartupPipelineRuntime({
|
||||||
createTerminalRuntime,
|
createTerminalRuntime,
|
||||||
|
createDictationRuntime,
|
||||||
createMessageStreamWsRuntime,
|
createMessageStreamWsRuntime,
|
||||||
createServerStartupRuntime,
|
createServerStartupRuntime,
|
||||||
});
|
});
|
||||||
@@ -1357,8 +1360,10 @@ async function main(options = {}) {
|
|||||||
tunnelRuntimeContext,
|
tunnelRuntimeContext,
|
||||||
attachSignals,
|
attachSignals,
|
||||||
apiOnly,
|
apiOnly,
|
||||||
|
dictationModelsDir: path.join(OPENCHAMBER_USER_CONFIG_ROOT, 'speech-models'),
|
||||||
});
|
});
|
||||||
terminalRuntime = startupPipelineResult.terminalRuntime;
|
terminalRuntime = startupPipelineResult.terminalRuntime;
|
||||||
|
dictationRuntime = startupPipelineResult.dictationRuntime;
|
||||||
messageStreamRuntime = startupPipelineResult.messageStreamRuntime;
|
messageStreamRuntime = startupPipelineResult.messageStreamRuntime;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -1397,6 +1402,11 @@ async function main(options = {}) {
|
|||||||
},
|
},
|
||||||
stop: (shutdownOptions = {}) => {
|
stop: (shutdownOptions = {}) => {
|
||||||
realtimeProxyRuntime.stop();
|
realtimeProxyRuntime.stop();
|
||||||
|
try {
|
||||||
|
dictationRuntime?.stop?.();
|
||||||
|
} catch {
|
||||||
|
// best-effort shutdown of the dictation worker
|
||||||
|
}
|
||||||
return gracefulShutdown({ exitProcess: shutdownOptions.exitProcess ?? false });
|
return gracefulShutdown({ exitProcess: shutdownOptions.exitProcess ?? false });
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
# Dictation module
|
||||||
|
|
||||||
|
Server-authoritative streaming speech-to-text for the chat composer, plus
|
||||||
|
local text-to-speech. The client streams 16 kHz mono PCM16 chunks (base64)
|
||||||
|
over a WebSocket; the server runs the transcription and streams live partial
|
||||||
|
transcripts back.
|
||||||
|
|
||||||
|
Local TTS (Kokoro via sherpa-onnx OfflineTts) runs in the same worker process
|
||||||
|
and is exposed as `POST /api/dictation/tts/speak` (JSON `{text, speakerId?,
|
||||||
|
speed?, model?}` → WAV bytes; 503 with `reasonCode` while the model is
|
||||||
|
downloading). TTS models live in the same catalog/downloader as STT models
|
||||||
|
(`local/model-catalog.js` `LOCAL_TTS_MODEL_CATALOG`) and are managed by the
|
||||||
|
same status/download/delete routes.
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
|
||||||
|
- `runtime.js` — registers `GET /api/dictation/status`,
|
||||||
|
`POST /api/dictation/models/:modelId/download`, and the
|
||||||
|
`/api/dictation/ws` WebSocket endpoint (auth-gated the same way as the
|
||||||
|
terminal WS: UI session token or `oc_url_token`, plus origin check).
|
||||||
|
Created from the startup pipeline (`startup-pipeline-runtime.js`) before
|
||||||
|
the generic OpenCode proxy so routes are not shadowed.
|
||||||
|
- `stream-manager.js` — `DictationStreamManager`, one per WS connection.
|
||||||
|
Chunk reordering by `seq` + ack, resampling to the provider rate,
|
||||||
|
auto-commit every ~15 s of audio, silence suppression by PCM peak,
|
||||||
|
partial-transcript concatenation, adaptive finalization timeout.
|
||||||
|
- `service.js` — provider resolution and readiness. Providers:
|
||||||
|
- `local` (default): sherpa-onnx Parakeet TDT in a forked worker process.
|
||||||
|
Models auto-download in the background on first use; while missing, the
|
||||||
|
stream fails with `reasonCode: 'model_download_in_progress'` and the
|
||||||
|
status route reports per-model install/download state.
|
||||||
|
- `openai-compatible`: buffered per-segment transcription against any
|
||||||
|
OpenAI-compatible `/v1/audio/transcriptions` endpoint
|
||||||
|
(`openai-compatible-session.js`, reuses `../tts/stt.js`).
|
||||||
|
- `local/` — worker process + client (IPC, idle shutdown TTL), sherpa
|
||||||
|
recognizer engine and realtime session (throttled re-decode for partials),
|
||||||
|
model catalog and downloader. The native `sherpa-onnx-node` addon is only
|
||||||
|
ever loaded inside the worker process.
|
||||||
|
- `audio.js` — PCM16 helpers: format parsing, peak, WAV wrapping, streaming
|
||||||
|
linear resampler.
|
||||||
|
|
||||||
|
## WebSocket protocol (JSON text frames)
|
||||||
|
|
||||||
|
Client → server: `start {dictationId, format, options}`,
|
||||||
|
`chunk {dictationId, seq, audio}`, `finish {dictationId, finalSeq}`,
|
||||||
|
`cancel {dictationId}`, `ping`.
|
||||||
|
|
||||||
|
Server → client: `ready`, `ack {ackSeq}`, `partial {text}`,
|
||||||
|
`finish_accepted {timeoutMs}`, `final {text}`,
|
||||||
|
`error {error, retryable, reasonCode?}`, `pong`.
|
||||||
|
|
||||||
|
`options` in `start` carries the client-selected provider config:
|
||||||
|
`{ provider: 'local' | 'openai-compatible', language?, localModel?,
|
||||||
|
openaiCompatible?: { baseUrl, model, apiKey } }`.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- Never load `sherpa-onnx-node` in the main server process.
|
||||||
|
- The stream manager acks only the highest contiguous seq; the client is
|
||||||
|
expected to retain unacked segments for retry/replay.
|
||||||
|
- Silence-only segments (peak < 300) are cleared, never committed, so
|
||||||
|
Whisper-style providers do not hallucinate on silence.
|
||||||
|
- Model files live under `~/.config/openchamber/speech-models`.
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
/**
|
||||||
|
* PCM16 audio helpers for the dictation streaming pipeline.
|
||||||
|
*
|
||||||
|
* All dictation audio travels as 16-bit little-endian mono PCM. The client
|
||||||
|
* captures at 16 kHz; providers may require a different rate, so chunks are
|
||||||
|
* resampled with Pcm16MonoResampler before being appended to an STT session.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the sample rate out of a format string like "audio/pcm;rate=16000;bits=16".
|
||||||
|
* @param {string} format
|
||||||
|
* @param {number|null} [fallback]
|
||||||
|
* @returns {number|null}
|
||||||
|
*/
|
||||||
|
export function parsePcmRateFromFormat(format, fallback = null) {
|
||||||
|
const match = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(String(format || ''));
|
||||||
|
if (!match) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
const rate = Number.parseInt(match[1], 10);
|
||||||
|
return Number.isFinite(rate) && rate > 0 ? rate : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return an Int16Array view over a PCM16LE buffer, copying when the buffer's
|
||||||
|
* byteOffset is not 2-byte aligned (IPC-transferred buffers can be views at
|
||||||
|
* odd offsets, and Int16Array requires an even start offset).
|
||||||
|
* @param {Buffer} pcm16le
|
||||||
|
* @returns {Int16Array}
|
||||||
|
*/
|
||||||
|
function toInt16Samples(pcm16le) {
|
||||||
|
if (pcm16le.byteOffset % 2 !== 0) {
|
||||||
|
const copy = Buffer.from(pcm16le);
|
||||||
|
return new Int16Array(copy.buffer, copy.byteOffset, copy.byteLength / 2);
|
||||||
|
}
|
||||||
|
return new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Peak absolute sample value of a PCM16LE buffer. Used for silence detection.
|
||||||
|
* @param {Buffer} pcm16le
|
||||||
|
* @returns {number}
|
||||||
|
*/
|
||||||
|
export function pcm16lePeakAbs(pcm16le) {
|
||||||
|
if (!pcm16le || pcm16le.length === 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (pcm16le.length % 2 !== 0) {
|
||||||
|
throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
|
||||||
|
}
|
||||||
|
const samples = toInt16Samples(pcm16le);
|
||||||
|
let peak = 0;
|
||||||
|
for (let i = 0; i < samples.length; i += 1) {
|
||||||
|
const v = samples[i];
|
||||||
|
const abs = v < 0 ? -v : v;
|
||||||
|
if (abs > peak) {
|
||||||
|
peak = abs;
|
||||||
|
if (peak >= 32767) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return peak;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert PCM16LE to Float32 samples in [-1, 1], with optional gain.
|
||||||
|
* @param {Buffer} pcm16le
|
||||||
|
* @param {number} [gain]
|
||||||
|
* @returns {Float32Array}
|
||||||
|
*/
|
||||||
|
export function pcm16leToFloat32(pcm16le, gain = 1) {
|
||||||
|
if (pcm16le.length % 2 !== 0) {
|
||||||
|
throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
|
||||||
|
}
|
||||||
|
const int16 = toInt16Samples(pcm16le);
|
||||||
|
const out = new Float32Array(int16.length);
|
||||||
|
for (let i = 0; i < int16.length; i += 1) {
|
||||||
|
const v = (int16[i] / 32768.0) * gain;
|
||||||
|
out[i] = Math.max(-1, Math.min(1, v));
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap raw PCM16LE mono audio in a WAV container.
|
||||||
|
* @param {Buffer} pcmBuffer
|
||||||
|
* @param {number} sampleRate
|
||||||
|
* @returns {Buffer}
|
||||||
|
*/
|
||||||
|
export function pcm16ToWav(pcmBuffer, sampleRate) {
|
||||||
|
const channels = 1;
|
||||||
|
const bitsPerSample = 16;
|
||||||
|
const headerSize = 44;
|
||||||
|
const wavBuffer = Buffer.alloc(headerSize + pcmBuffer.length);
|
||||||
|
const byteRate = (sampleRate * channels * bitsPerSample) / 8;
|
||||||
|
const blockAlign = (channels * bitsPerSample) / 8;
|
||||||
|
|
||||||
|
wavBuffer.write('RIFF', 0);
|
||||||
|
wavBuffer.writeUInt32LE(36 + pcmBuffer.length, 4);
|
||||||
|
wavBuffer.write('WAVE', 8);
|
||||||
|
wavBuffer.write('fmt ', 12);
|
||||||
|
wavBuffer.writeUInt32LE(16, 16);
|
||||||
|
wavBuffer.writeUInt16LE(1, 20);
|
||||||
|
wavBuffer.writeUInt16LE(channels, 22);
|
||||||
|
wavBuffer.writeUInt32LE(sampleRate, 24);
|
||||||
|
wavBuffer.writeUInt32LE(byteRate, 28);
|
||||||
|
wavBuffer.writeUInt16LE(blockAlign, 32);
|
||||||
|
wavBuffer.writeUInt16LE(bitsPerSample, 34);
|
||||||
|
wavBuffer.write('data', 36);
|
||||||
|
wavBuffer.writeUInt32LE(pcmBuffer.length, 40);
|
||||||
|
pcmBuffer.copy(wavBuffer, 44);
|
||||||
|
|
||||||
|
return wavBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Streaming linear-interpolation resampler for PCM16LE mono audio.
|
||||||
|
* Carries one sample across chunk boundaries so consecutive chunks resample
|
||||||
|
* without seams.
|
||||||
|
*/
|
||||||
|
export class Pcm16MonoResampler {
|
||||||
|
/**
|
||||||
|
* @param {{ inputRate: number, outputRate: number }} params
|
||||||
|
*/
|
||||||
|
constructor({ inputRate, outputRate }) {
|
||||||
|
this.inputRate = inputRate;
|
||||||
|
this.outputRate = outputRate;
|
||||||
|
this.step = inputRate / outputRate;
|
||||||
|
this.pos = 0;
|
||||||
|
this.carrySample = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
reset() {
|
||||||
|
this.pos = 0;
|
||||||
|
this.carrySample = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {Buffer} pcm16le
|
||||||
|
* @returns {Buffer}
|
||||||
|
*/
|
||||||
|
processChunk(pcm16le) {
|
||||||
|
if (pcm16le.length === 0) {
|
||||||
|
return Buffer.alloc(0);
|
||||||
|
}
|
||||||
|
if (pcm16le.length % 2 !== 0) {
|
||||||
|
throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const srcChunk = toInt16Samples(pcm16le);
|
||||||
|
|
||||||
|
const hasCarry = this.carrySample !== null;
|
||||||
|
const srcLen = srcChunk.length + (hasCarry ? 1 : 0);
|
||||||
|
if (srcLen < 2) {
|
||||||
|
this.carrySample = srcChunk.length ? srcChunk[srcChunk.length - 1] : this.carrySample;
|
||||||
|
return Buffer.alloc(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const src = new Float32Array(srcLen);
|
||||||
|
let offset = 0;
|
||||||
|
if (hasCarry) {
|
||||||
|
src[0] = this.carrySample / 32768;
|
||||||
|
offset = 1;
|
||||||
|
}
|
||||||
|
for (let i = 0; i < srcChunk.length; i += 1) {
|
||||||
|
src[offset + i] = srcChunk[i] / 32768;
|
||||||
|
}
|
||||||
|
|
||||||
|
const out = [];
|
||||||
|
const maxPos = src.length - 1;
|
||||||
|
|
||||||
|
while (this.pos < maxPos) {
|
||||||
|
const i = Math.floor(this.pos);
|
||||||
|
const frac = this.pos - i;
|
||||||
|
const s0 = src[i];
|
||||||
|
const s1 = src[i + 1];
|
||||||
|
const sample = s0 + (s1 - s0) * frac;
|
||||||
|
const clamped = Math.max(-1, Math.min(1, sample));
|
||||||
|
out.push(Math.round(clamped * 32767));
|
||||||
|
this.pos += this.step;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.carrySample = srcChunk[srcChunk.length - 1];
|
||||||
|
|
||||||
|
const shift = src.length - 1;
|
||||||
|
this.pos = this.pos - shift;
|
||||||
|
if (this.pos < 0) {
|
||||||
|
this.pos = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const outArr = Int16Array.from(out);
|
||||||
|
return Buffer.from(outArr.buffer, outArr.byteOffset, outArr.byteLength);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
/**
|
||||||
|
* Catalog of local sherpa-onnx STT models available for dictation.
|
||||||
|
* Models are downloaded on demand from the k2-fsa GitHub releases and
|
||||||
|
* extracted under the OpenChamber speech-models directory.
|
||||||
|
*
|
||||||
|
* `type` selects the recognizer construction path in the worker:
|
||||||
|
* - 'nemo_transducer': encoder/decoder/joiner transducer (Parakeet)
|
||||||
|
* - 'whisper': encoder/decoder Whisper export
|
||||||
|
* `files` maps logical roles to file names inside the extracted directory.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
export const LOCAL_STT_MODEL_CATALOG = {
|
||||||
|
'parakeet-tdt-0.6b-v2-int8': {
|
||||||
|
type: 'nemo_transducer',
|
||||||
|
archiveUrl:
|
||||||
|
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2',
|
||||||
|
extractedDir: 'sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8',
|
||||||
|
files: {
|
||||||
|
encoder: 'encoder.int8.onnx',
|
||||||
|
decoder: 'decoder.int8.onnx',
|
||||||
|
joiner: 'joiner.int8.onnx',
|
||||||
|
tokens: 'tokens.txt',
|
||||||
|
},
|
||||||
|
description: 'NVIDIA Parakeet TDT v2 (English)',
|
||||||
|
},
|
||||||
|
'parakeet-tdt-0.6b-v3-int8': {
|
||||||
|
type: 'nemo_transducer',
|
||||||
|
archiveUrl:
|
||||||
|
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2',
|
||||||
|
extractedDir: 'sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8',
|
||||||
|
files: {
|
||||||
|
encoder: 'encoder.int8.onnx',
|
||||||
|
decoder: 'decoder.int8.onnx',
|
||||||
|
joiner: 'joiner.int8.onnx',
|
||||||
|
tokens: 'tokens.txt',
|
||||||
|
},
|
||||||
|
description: 'NVIDIA Parakeet TDT v3 (25 European languages, auto-detected)',
|
||||||
|
},
|
||||||
|
'whisper-base-int8': {
|
||||||
|
type: 'whisper',
|
||||||
|
archiveUrl:
|
||||||
|
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-base.tar.bz2',
|
||||||
|
extractedDir: 'sherpa-onnx-whisper-base',
|
||||||
|
files: {
|
||||||
|
encoder: 'base-encoder.int8.onnx',
|
||||||
|
decoder: 'base-decoder.int8.onnx',
|
||||||
|
tokens: 'base-tokens.txt',
|
||||||
|
},
|
||||||
|
description: 'OpenAI Whisper base (multilingual, smaller and lighter)',
|
||||||
|
},
|
||||||
|
'whisper-tiny-int8': {
|
||||||
|
type: 'whisper',
|
||||||
|
archiveUrl:
|
||||||
|
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2',
|
||||||
|
extractedDir: 'sherpa-onnx-whisper-tiny',
|
||||||
|
files: {
|
||||||
|
encoder: 'tiny-encoder.int8.onnx',
|
||||||
|
decoder: 'tiny-decoder.int8.onnx',
|
||||||
|
tokens: 'tiny-tokens.txt',
|
||||||
|
},
|
||||||
|
description: 'OpenAI Whisper tiny (multilingual, fastest and lightest)',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Local text-to-speech models (sherpa-onnx OfflineTts). Downloaded and
|
||||||
|
* managed through the same pipeline as the STT models.
|
||||||
|
*/
|
||||||
|
export const LOCAL_TTS_MODEL_CATALOG = {
|
||||||
|
'kokoro-en-v0_19': {
|
||||||
|
type: 'kokoro',
|
||||||
|
archiveUrl:
|
||||||
|
'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2',
|
||||||
|
extractedDir: 'kokoro-en-v0_19',
|
||||||
|
files: {
|
||||||
|
model: 'model.onnx',
|
||||||
|
voices: 'voices.bin',
|
||||||
|
tokens: 'tokens.txt',
|
||||||
|
espeakData: 'espeak-ng-data',
|
||||||
|
},
|
||||||
|
description: 'Kokoro TTS (English, natural voices)',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export const DEFAULT_LOCAL_STT_MODEL = 'parakeet-tdt-0.6b-v2-int8';
|
||||||
|
export const DEFAULT_LOCAL_TTS_MODEL = 'kokoro-en-v0_19';
|
||||||
|
|
||||||
|
export const LOCAL_STT_MODEL_IDS = Object.keys(LOCAL_STT_MODEL_CATALOG);
|
||||||
|
export const LOCAL_TTS_MODEL_IDS = Object.keys(LOCAL_TTS_MODEL_CATALOG);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} modelId
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function isLocalSttModelId(modelId) {
|
||||||
|
return typeof modelId === 'string' && Object.hasOwn(LOCAL_STT_MODEL_CATALOG, modelId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} modelId
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function isLocalTtsModelId(modelId) {
|
||||||
|
return typeof modelId === 'string' && Object.hasOwn(LOCAL_TTS_MODEL_CATALOG, modelId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Any managed local model (STT or TTS).
|
||||||
|
* @param {string} modelId
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function isLocalModelId(modelId) {
|
||||||
|
return isLocalSttModelId(modelId) || isLocalTtsModelId(modelId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Spec lookup across both catalogs (STT and TTS).
|
||||||
|
* @param {string} modelId
|
||||||
|
*/
|
||||||
|
export function getLocalSttModelSpec(modelId) {
|
||||||
|
const spec = LOCAL_STT_MODEL_CATALOG[modelId] ?? LOCAL_TTS_MODEL_CATALOG[modelId];
|
||||||
|
if (!spec) {
|
||||||
|
throw new Error(`Unknown local speech model id: ${modelId}`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
id: modelId,
|
||||||
|
...spec,
|
||||||
|
requiredFiles: Object.values(spec.files),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} modelsDir
|
||||||
|
* @param {string} modelId
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
export function getLocalSttModelDir(modelsDir, modelId) {
|
||||||
|
return path.join(modelsDir, getLocalSttModelSpec(modelId).extractedDir);
|
||||||
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
/**
|
||||||
|
* Downloads and extracts local sherpa-onnx STT model archives.
|
||||||
|
* Archives (.tar.bz2) come from the k2-fsa GitHub releases and are extracted
|
||||||
|
* with the system `tar` into the speech-models directory.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createWriteStream } from 'fs';
|
||||||
|
import { mkdir, rename, rm, stat } from 'fs/promises';
|
||||||
|
import path from 'path';
|
||||||
|
import { Readable } from 'stream';
|
||||||
|
import { pipeline } from 'stream/promises';
|
||||||
|
import { spawn } from 'child_process';
|
||||||
|
|
||||||
|
import { getLocalSttModelSpec } from './model-catalog.js';
|
||||||
|
|
||||||
|
async function hasRequiredFiles(modelDir, requiredFiles) {
|
||||||
|
const results = await Promise.all(
|
||||||
|
requiredFiles.map(async (rel) => {
|
||||||
|
try {
|
||||||
|
const s = await stat(path.join(modelDir, rel));
|
||||||
|
if (s.isDirectory()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return s.isFile() && s.size > 0;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return results.every(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downloadToFile(url, outputPath, onProgress) {
|
||||||
|
const res = await fetch(url);
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`);
|
||||||
|
}
|
||||||
|
if (!res.body) {
|
||||||
|
throw new Error(`Failed to download ${url}: missing response body`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalBytes = Number.parseInt(res.headers.get('content-length') || '', 10) || null;
|
||||||
|
let downloadedBytes = 0;
|
||||||
|
|
||||||
|
const tmpPath = `${outputPath}.tmp-${Date.now()}`;
|
||||||
|
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||||
|
|
||||||
|
const nodeStream = Readable.fromWeb(res.body);
|
||||||
|
if (typeof onProgress === 'function') {
|
||||||
|
nodeStream.on('data', (chunk) => {
|
||||||
|
downloadedBytes += chunk.length;
|
||||||
|
onProgress(downloadedBytes, totalBytes);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await pipeline(nodeStream, createWriteStream(tmpPath));
|
||||||
|
await rename(tmpPath, outputPath);
|
||||||
|
} catch (error) {
|
||||||
|
await rm(tmpPath, { force: true }).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractTarArchive(archivePath, destDir) {
|
||||||
|
await mkdir(destDir, { recursive: true });
|
||||||
|
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
const child = spawn('tar', ['xf', archivePath, '-C', destDir], {
|
||||||
|
stdio: 'ignore',
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
child.on('error', reject);
|
||||||
|
child.on('exit', (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
resolve();
|
||||||
|
} else {
|
||||||
|
reject(new Error(`tar exited with code ${code}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function isNonEmptyFile(filePath) {
|
||||||
|
try {
|
||||||
|
const s = await stat(filePath);
|
||||||
|
return s.isFile() && s.size > 0;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether a model is fully installed (all required files present).
|
||||||
|
* @param {string} modelsDir
|
||||||
|
* @param {string} modelId
|
||||||
|
* @returns {Promise<boolean>}
|
||||||
|
*/
|
||||||
|
export async function isLocalSttModelInstalled(modelsDir, modelId) {
|
||||||
|
const spec = getLocalSttModelSpec(modelId);
|
||||||
|
return hasRequiredFiles(path.join(modelsDir, spec.extractedDir), spec.requiredFiles);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure a model is downloaded and extracted. Resolves with the model dir.
|
||||||
|
*
|
||||||
|
* Extraction is staged: the archive unpacks into a temporary directory and is
|
||||||
|
* verified before being renamed into place. An interrupted or failed tar must
|
||||||
|
* never leave partial files at the final path — the installed check only
|
||||||
|
* verifies file presence, so a truncated .onnx there would be treated as an
|
||||||
|
* installed model forever ("Protobuf parsing failed" at load time).
|
||||||
|
*
|
||||||
|
* @param {{ modelsDir: string, modelId: string,
|
||||||
|
* onProgress?: (downloadedBytes: number, totalBytes: number | null) => void }} options
|
||||||
|
* @returns {Promise<string>}
|
||||||
|
*/
|
||||||
|
export async function ensureLocalSttModel({ modelsDir, modelId, onProgress }) {
|
||||||
|
const spec = getLocalSttModelSpec(modelId);
|
||||||
|
const modelDir = path.join(modelsDir, spec.extractedDir);
|
||||||
|
if (await hasRequiredFiles(modelDir, spec.requiredFiles)) {
|
||||||
|
return modelDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A directory that exists but fails the required-files check is a partial
|
||||||
|
// extraction from an earlier interrupted attempt — remove it before retrying.
|
||||||
|
await rm(modelDir, { recursive: true, force: true }).catch(() => undefined);
|
||||||
|
|
||||||
|
const downloadsDir = path.join(modelsDir, '.downloads');
|
||||||
|
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||||
|
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||||
|
|
||||||
|
if (!(await isNonEmptyFile(archivePath))) {
|
||||||
|
await downloadToFile(spec.archiveUrl, archivePath, onProgress);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stagingDir = path.join(modelsDir, `.staging-${spec.extractedDir}-${Date.now()}`);
|
||||||
|
try {
|
||||||
|
await extractTarArchive(archivePath, stagingDir);
|
||||||
|
|
||||||
|
const stagedModelDir = path.join(stagingDir, spec.extractedDir);
|
||||||
|
if (!(await hasRequiredFiles(stagedModelDir, spec.requiredFiles))) {
|
||||||
|
// Bad archive (truncated download / corrupt cache): drop it so the next
|
||||||
|
// attempt re-downloads instead of re-extracting the same broken bytes.
|
||||||
|
await rm(archivePath, { force: true }).catch(() => undefined);
|
||||||
|
throw new Error(
|
||||||
|
`Extracted ${archiveFilename}, but required model files are missing or empty. The archive was discarded; retry to re-download.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await rename(stagedModelDir, modelDir);
|
||||||
|
} catch (error) {
|
||||||
|
await rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||||
|
// Any extraction failure means the cached archive can't be trusted
|
||||||
|
// (corrupt bz2, truncated download). Discard it so retry re-downloads.
|
||||||
|
await rm(archivePath, { force: true }).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
await rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||||
|
|
||||||
|
await rm(archivePath, { force: true }).catch(() => undefined);
|
||||||
|
|
||||||
|
return modelDir;
|
||||||
|
}
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
/**
|
||||||
|
* Loader for the sherpa-onnx-node native addon.
|
||||||
|
*
|
||||||
|
* sherpa-onnx-node ships its native addon and shared libraries in a
|
||||||
|
* platform-specific package (e.g. sherpa-onnx-darwin-arm64). The shared
|
||||||
|
* libraries must be findable via the platform's dynamic-loader search path,
|
||||||
|
* so the loader prepends the platform package directory to LD_LIBRARY_PATH /
|
||||||
|
* DYLD_LIBRARY_PATH / PATH before requiring the addon.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createRequire } from 'module';
|
||||||
|
import path from 'path';
|
||||||
|
import { existsSync } from 'fs';
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
|
let cached = null;
|
||||||
|
|
||||||
|
function sherpaPlatformPackageName(platform = process.platform, arch = process.arch) {
|
||||||
|
const normalizedPlatform = platform === 'win32' ? 'win' : platform;
|
||||||
|
return `sherpa-onnx-${normalizedPlatform}-${arch}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sherpaLoaderEnvKey(platform = process.platform) {
|
||||||
|
if (platform === 'linux') {
|
||||||
|
return 'LD_LIBRARY_PATH';
|
||||||
|
}
|
||||||
|
if (platform === 'darwin') {
|
||||||
|
return 'DYLD_LIBRARY_PATH';
|
||||||
|
}
|
||||||
|
if (platform === 'win32') {
|
||||||
|
return 'PATH';
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function prependEnvPath(existing, value) {
|
||||||
|
const parts = String(existing ?? '').split(path.delimiter).filter(Boolean);
|
||||||
|
if (parts.includes(value)) {
|
||||||
|
return parts.join(path.delimiter);
|
||||||
|
}
|
||||||
|
return [value, ...parts].join(path.delimiter);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Case-insensitive env key lookup: on Windows `{...process.env}` yields a
|
||||||
|
* plain object where PATH may be stored as `Path`. Using a hardcoded 'PATH'
|
||||||
|
* would create a duplicate key and break the child process PATH.
|
||||||
|
*/
|
||||||
|
function findEnvKey(env, key) {
|
||||||
|
const lower = key.toLowerCase();
|
||||||
|
for (const k of Object.keys(env)) {
|
||||||
|
if (k.toLowerCase() === lower) {
|
||||||
|
return k;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveSherpaLibDir(platform = process.platform, arch = process.arch) {
|
||||||
|
const packageName = sherpaPlatformPackageName(platform, arch);
|
||||||
|
try {
|
||||||
|
const pkgJson = require.resolve(`${packageName}/package.json`);
|
||||||
|
// Electron packages node_modules inside app.asar, but native addons and
|
||||||
|
// their shared libraries are extracted to app.asar.unpacked. The dynamic
|
||||||
|
// loader (dlopen/DYLD/LD) cannot read from the asar archive, so point the
|
||||||
|
// search path at the unpacked copy.
|
||||||
|
const dir = path.dirname(pkgJson);
|
||||||
|
const unpacked = dir.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`);
|
||||||
|
return existsSync(unpacked) ? unpacked : dir;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prepend the sherpa platform package dir to the loader search path env var.
|
||||||
|
* Mutates the provided env object.
|
||||||
|
* @param {NodeJS.ProcessEnv} env
|
||||||
|
*/
|
||||||
|
export function applySherpaLoaderEnv(env) {
|
||||||
|
const key = sherpaLoaderEnvKey();
|
||||||
|
const libDir = resolveSherpaLibDir();
|
||||||
|
if (!key || !libDir) {
|
||||||
|
return { key: null, libDir: null };
|
||||||
|
}
|
||||||
|
const actualKey = findEnvKey(env, key);
|
||||||
|
env[actualKey] = prependEnvPath(env[actualKey], libDir);
|
||||||
|
return { key, libDir };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load the sherpa-onnx-node module, trying the upstream entry first and then
|
||||||
|
* the platform addon directly.
|
||||||
|
*/
|
||||||
|
export function loadSherpaOnnxNode() {
|
||||||
|
if (cached) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
const attempts = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
cached = require('sherpa-onnx-node');
|
||||||
|
return cached;
|
||||||
|
} catch (error) {
|
||||||
|
attempts.push(`sherpa-onnx-node: ${error?.message || String(error)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const libDir = resolveSherpaLibDir();
|
||||||
|
if (libDir) {
|
||||||
|
applySherpaLoaderEnv(process.env);
|
||||||
|
const addonPath = path.join(libDir, 'sherpa-onnx.node');
|
||||||
|
if (existsSync(addonPath)) {
|
||||||
|
try {
|
||||||
|
cached = require(addonPath);
|
||||||
|
return cached;
|
||||||
|
} catch (error) {
|
||||||
|
attempts.push(`${addonPath}: ${error?.message || String(error)}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
attempts.push(`${addonPath}: file not found`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
attempts.push(`${sherpaPlatformPackageName()}: platform package not installed`);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
[
|
||||||
|
`Failed to load sherpa-onnx-node for ${process.platform}-${process.arch}.`,
|
||||||
|
`Node ${process.version} (ABI ${process.versions.modules}).`,
|
||||||
|
'Load attempts:',
|
||||||
|
...attempts.map((line) => `- ${line}`),
|
||||||
|
].join('\n'),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
/**
|
||||||
|
* Sherpa-onnx offline recognizer engine (NeMo transducer / Parakeet) plus a
|
||||||
|
* realtime streaming transcription session that re-decodes the accumulated
|
||||||
|
* segment audio on a throttle to produce live partial transcripts.
|
||||||
|
*
|
||||||
|
* Runs inside the dictation worker process only — never load the native
|
||||||
|
* addon in the main server process.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { EventEmitter } from 'events';
|
||||||
|
import { existsSync } from 'fs';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
|
||||||
|
import { loadSherpaOnnxNode } from './sherpa-loader.js';
|
||||||
|
import { pcm16lePeakAbs, pcm16leToFloat32 } from '../audio.js';
|
||||||
|
|
||||||
|
function assertFileExists(filePath, label) {
|
||||||
|
if (!existsSync(filePath)) {
|
||||||
|
throw new Error(`Missing ${label}: ${filePath}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SherpaOfflineRecognizerEngine {
|
||||||
|
/**
|
||||||
|
* @param {{ type: 'nemo_transducer' | 'whisper',
|
||||||
|
* encoder: string, decoder: string, joiner?: string, tokens: string,
|
||||||
|
* numThreads?: number }} config
|
||||||
|
*/
|
||||||
|
constructor(config) {
|
||||||
|
assertFileExists(config.encoder, 'offline encoder');
|
||||||
|
assertFileExists(config.decoder, 'offline decoder');
|
||||||
|
if (config.type === 'nemo_transducer') {
|
||||||
|
assertFileExists(config.joiner, 'offline joiner');
|
||||||
|
}
|
||||||
|
assertFileExists(config.tokens, 'tokens');
|
||||||
|
|
||||||
|
const sherpa = loadSherpaOnnxNode();
|
||||||
|
|
||||||
|
const modelConfig =
|
||||||
|
config.type === 'whisper'
|
||||||
|
? {
|
||||||
|
whisper: {
|
||||||
|
encoder: config.encoder,
|
||||||
|
decoder: config.decoder,
|
||||||
|
// Empty language auto-detects for multilingual Whisper exports.
|
||||||
|
language: '',
|
||||||
|
task: 'transcribe',
|
||||||
|
tailPaddings: -1,
|
||||||
|
},
|
||||||
|
tokens: config.tokens,
|
||||||
|
modelType: 'whisper',
|
||||||
|
numThreads: config.numThreads ?? 2,
|
||||||
|
provider: 'cpu',
|
||||||
|
debug: 0,
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
transducer: {
|
||||||
|
encoder: config.encoder,
|
||||||
|
decoder: config.decoder,
|
||||||
|
joiner: config.joiner,
|
||||||
|
},
|
||||||
|
tokens: config.tokens,
|
||||||
|
modelType: 'nemo_transducer',
|
||||||
|
numThreads: config.numThreads ?? 2,
|
||||||
|
provider: 'cpu',
|
||||||
|
debug: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const recognizerConfig = {
|
||||||
|
featConfig: {
|
||||||
|
sampleRate: 16000,
|
||||||
|
featureDim: 80,
|
||||||
|
},
|
||||||
|
modelConfig,
|
||||||
|
decodingMethod: 'greedy_search',
|
||||||
|
maxActivePaths: 4,
|
||||||
|
};
|
||||||
|
|
||||||
|
this.recognizer = new sherpa.OfflineRecognizer(recognizerConfig);
|
||||||
|
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
||||||
|
this.sampleRate =
|
||||||
|
typeof sr === 'number' && Number.isFinite(sr) && sr > 0
|
||||||
|
? sr
|
||||||
|
: recognizerConfig.featConfig.sampleRate;
|
||||||
|
}
|
||||||
|
|
||||||
|
createStream() {
|
||||||
|
return this.recognizer.createStream();
|
||||||
|
}
|
||||||
|
|
||||||
|
acceptWaveform(stream, sampleRate, samples) {
|
||||||
|
if (!stream || typeof stream.acceptWaveform !== 'function') {
|
||||||
|
throw new Error('Unexpected sherpa offline stream: missing acceptWaveform()');
|
||||||
|
}
|
||||||
|
// sherpa-onnx-node expects acceptWaveform({ samples, sampleRate });
|
||||||
|
// the WASM build expects acceptWaveform(sampleRate, samples).
|
||||||
|
if (stream.acceptWaveform.length <= 1) {
|
||||||
|
stream.acceptWaveform({ samples, sampleRate });
|
||||||
|
} else {
|
||||||
|
stream.acceptWaveform(sampleRate, samples);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decode a full PCM16 segment and return its text.
|
||||||
|
* Applies auto-gain when the peak is low so quiet microphones still decode.
|
||||||
|
* @param {Buffer} pcm16
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
decodePcm16(pcm16) {
|
||||||
|
if (pcm16.length === 0) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
const peak = pcm16lePeakAbs(pcm16);
|
||||||
|
const peakFloat = peak / 32768.0;
|
||||||
|
const targetPeak = 0.6;
|
||||||
|
const maxGain = 50;
|
||||||
|
const gain =
|
||||||
|
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||||
|
|
||||||
|
const stream = this.createStream();
|
||||||
|
try {
|
||||||
|
const floatSamples = pcm16leToFloat32(pcm16, gain);
|
||||||
|
this.acceptWaveform(stream, this.sampleRate, floatSamples);
|
||||||
|
this.recognizer.decode(stream);
|
||||||
|
const result = this.recognizer.getResult(stream);
|
||||||
|
const text =
|
||||||
|
typeof result === 'object' && result && 'text' in result ? result.text : result;
|
||||||
|
return String(text ?? '').trim();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
stream.free?.();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
free() {
|
||||||
|
try {
|
||||||
|
this.recognizer?.free?.();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Streaming transcription session backed by the offline recognizer.
|
||||||
|
* Accumulates the current segment's PCM and re-decodes it at most every
|
||||||
|
* `minDecodeIntervalMs` to emit non-final partial transcripts; `commit()`
|
||||||
|
* finalizes the segment and starts a new one.
|
||||||
|
*
|
||||||
|
* Implements the StreamingTranscriptionSession contract used by
|
||||||
|
* DictationStreamManager.
|
||||||
|
*/
|
||||||
|
export class SherpaRealtimeTranscriptionSession extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* @param {{ engine: SherpaOfflineRecognizerEngine, minDecodeIntervalMs?: number }} params
|
||||||
|
*/
|
||||||
|
constructor({ engine, minDecodeIntervalMs }) {
|
||||||
|
super();
|
||||||
|
this.engine = engine;
|
||||||
|
this.requiredSampleRate = engine.sampleRate;
|
||||||
|
this.minDecodeIntervalMs = minDecodeIntervalMs ?? 350;
|
||||||
|
this.connected = false;
|
||||||
|
this.currentSegmentId = null;
|
||||||
|
this.previousSegmentId = null;
|
||||||
|
this.lastPartialText = '';
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
this.lastDecodeAt = 0;
|
||||||
|
this.decoding = false;
|
||||||
|
this.pendingDecode = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async connect() {
|
||||||
|
if (this.connected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.currentSegmentId = randomUUID();
|
||||||
|
this.connected = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
appendPcm16(chunk) {
|
||||||
|
if (!this.connected || !this.currentSegmentId) {
|
||||||
|
this.emit('error', new Error('Sherpa realtime session not connected'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
|
||||||
|
this.maybeDecode(false).catch((err) => {
|
||||||
|
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
commit() {
|
||||||
|
if (!this.connected || !this.currentSegmentId) {
|
||||||
|
this.emit('error', new Error('Sherpa realtime session not connected'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
await this.maybeDecode(true);
|
||||||
|
const finalText = this.lastPartialText;
|
||||||
|
const segmentId = this.currentSegmentId;
|
||||||
|
const previousSegmentId = this.previousSegmentId;
|
||||||
|
|
||||||
|
this.emit('committed', { segmentId, previousSegmentId });
|
||||||
|
this.emit('transcript', { segmentId, transcript: finalText, isFinal: true });
|
||||||
|
|
||||||
|
this.previousSegmentId = segmentId;
|
||||||
|
this.currentSegmentId = randomUUID();
|
||||||
|
this.lastPartialText = '';
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
} catch (err) {
|
||||||
|
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
if (!this.connected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
this.currentSegmentId = randomUUID();
|
||||||
|
this.lastPartialText = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
close() {
|
||||||
|
this.connected = false;
|
||||||
|
this.currentSegmentId = null;
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async maybeDecode(force) {
|
||||||
|
if (!this.connected || !this.currentSegmentId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.decoding) {
|
||||||
|
this.pendingDecode = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.decoding = true;
|
||||||
|
try {
|
||||||
|
const decodeStartedAt = Date.now();
|
||||||
|
const text = this.engine.decodePcm16(this.pcm16);
|
||||||
|
this.lastDecodeAt = Date.now();
|
||||||
|
// Adaptive throttle: on slow hardware (or heavy models) re-decoding the
|
||||||
|
// growing segment every 350ms would monopolize the worker. Space partial
|
||||||
|
// decodes to ~1.5x the observed decode time.
|
||||||
|
this.minDecodeIntervalMs = Math.max(350, (this.lastDecodeAt - decodeStartedAt) * 1.5);
|
||||||
|
if (text !== this.lastPartialText) {
|
||||||
|
this.lastPartialText = text;
|
||||||
|
this.emit('transcript', {
|
||||||
|
segmentId: this.currentSegmentId,
|
||||||
|
transcript: text,
|
||||||
|
isFinal: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
this.decoding = false;
|
||||||
|
if (this.pendingDecode) {
|
||||||
|
this.pendingDecode = false;
|
||||||
|
await this.maybeDecode(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
/**
|
||||||
|
* Sherpa-onnx offline TTS (Kokoro). Runs inside the dictation worker process
|
||||||
|
* only — never load the native addon in the main server process.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { existsSync } from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { loadSherpaOnnxNode } from './sherpa-loader.js';
|
||||||
|
|
||||||
|
function assertFileExists(filePath, label) {
|
||||||
|
if (!existsSync(filePath)) {
|
||||||
|
throw new Error(`Missing ${label}: ${filePath}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function float32ToPcm16le(samples) {
|
||||||
|
const out = new Int16Array(samples.length);
|
||||||
|
for (let i = 0; i < samples.length; i += 1) {
|
||||||
|
const clamped = Math.max(-1, Math.min(1, samples[i]));
|
||||||
|
out[i] = Math.round(clamped * 32767);
|
||||||
|
}
|
||||||
|
return Buffer.from(out.buffer, out.byteOffset, out.byteLength);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SherpaTtsEngine {
|
||||||
|
/**
|
||||||
|
* @param {{ modelDir: string, files: { model: string, voices: string, tokens: string, espeakData: string }, numThreads?: number }} config
|
||||||
|
*/
|
||||||
|
constructor(config) {
|
||||||
|
const modelPath = path.join(config.modelDir, config.files.model);
|
||||||
|
const voicesPath = path.join(config.modelDir, config.files.voices);
|
||||||
|
const tokensPath = path.join(config.modelDir, config.files.tokens);
|
||||||
|
const dataDir = path.join(config.modelDir, config.files.espeakData);
|
||||||
|
|
||||||
|
assertFileExists(modelPath, 'TTS model');
|
||||||
|
assertFileExists(voicesPath, 'TTS voices');
|
||||||
|
assertFileExists(tokensPath, 'TTS tokens');
|
||||||
|
assertFileExists(dataDir, 'TTS espeak-ng dataDir');
|
||||||
|
|
||||||
|
const sherpa = loadSherpaOnnxNode();
|
||||||
|
if (typeof sherpa.OfflineTts !== 'function') {
|
||||||
|
throw new Error('sherpa-onnx-node OfflineTts is unavailable');
|
||||||
|
}
|
||||||
|
|
||||||
|
this.tts = new sherpa.OfflineTts({
|
||||||
|
model: {
|
||||||
|
kokoro: {
|
||||||
|
model: modelPath,
|
||||||
|
voices: voicesPath,
|
||||||
|
tokens: tokensPath,
|
||||||
|
dataDir,
|
||||||
|
lengthScale: 1.0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
numThreads: config.numThreads ?? 2,
|
||||||
|
provider: 'cpu',
|
||||||
|
maxNumSentences: 1,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synthesize text to PCM16LE.
|
||||||
|
* @param {string} text
|
||||||
|
* @param {{ speakerId?: number, speed?: number }} [options]
|
||||||
|
* @returns {{ pcm16: Buffer, sampleRate: number }}
|
||||||
|
*/
|
||||||
|
synthesize(text, options = {}) {
|
||||||
|
const trimmed = String(text || '').trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
throw new Error('Cannot synthesize empty text');
|
||||||
|
}
|
||||||
|
|
||||||
|
const audio = this.tts.generate({
|
||||||
|
text: trimmed,
|
||||||
|
sid: Number.isInteger(options.speakerId) ? options.speakerId : 0,
|
||||||
|
speed: typeof options.speed === 'number' && options.speed > 0 ? options.speed : 1.0,
|
||||||
|
// Request a copied buffer from sherpa itself: native external-backed
|
||||||
|
// typed arrays are rejected by Electron.
|
||||||
|
enableExternalBuffer: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
let samples = null;
|
||||||
|
if (audio && audio.samples instanceof Float32Array) {
|
||||||
|
samples = Float32Array.from(audio.samples);
|
||||||
|
} else if (audio && Array.isArray(audio.samples)) {
|
||||||
|
samples = Float32Array.from(audio.samples);
|
||||||
|
}
|
||||||
|
if (!samples) {
|
||||||
|
throw new Error('Unexpected sherpa TTS output: missing Float32 samples');
|
||||||
|
}
|
||||||
|
|
||||||
|
const sampleRate =
|
||||||
|
audio && typeof audio.sampleRate === 'number' && audio.sampleRate > 0
|
||||||
|
? audio.sampleRate
|
||||||
|
: typeof this.tts.sampleRate === 'number' && this.tts.sampleRate > 0
|
||||||
|
? this.tts.sampleRate
|
||||||
|
: 24000;
|
||||||
|
|
||||||
|
return { pcm16: float32ToPcm16le(samples), sampleRate };
|
||||||
|
}
|
||||||
|
|
||||||
|
free() {
|
||||||
|
try {
|
||||||
|
this.tts?.free?.();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,352 @@
|
|||||||
|
/**
|
||||||
|
* Client for the dictation local-speech worker process.
|
||||||
|
*
|
||||||
|
* Lazily forks the worker on first use, correlates request/response messages
|
||||||
|
* by requestId, routes session events to per-session EventEmitters, and
|
||||||
|
* shuts the worker down after an idle TTL so the ONNX runtime does not sit
|
||||||
|
* in memory while dictation is unused.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { fork } from 'child_process';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { EventEmitter } from 'events';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
|
import { applySherpaLoaderEnv } from './sherpa-loader.js';
|
||||||
|
|
||||||
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
|
||||||
|
const DEFAULT_IDLE_TTL_MS = 5 * 60 * 1000;
|
||||||
|
const DEFAULT_LOCAL_SAMPLE_RATE = 16000;
|
||||||
|
const STDERR_TAIL_MAX_CHARS = 2000;
|
||||||
|
|
||||||
|
function forkDictationWorker() {
|
||||||
|
const env = { ...process.env };
|
||||||
|
applySherpaLoaderEnv(env);
|
||||||
|
return fork(fileURLToPath(new URL('./worker-process.js', import.meta.url)), [], {
|
||||||
|
env,
|
||||||
|
serialization: 'advanced',
|
||||||
|
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DictationWorkerClient {
|
||||||
|
/**
|
||||||
|
* @param {{ requestTimeoutMs?: number, idleTtlMs?: number }} [options]
|
||||||
|
*/
|
||||||
|
constructor(options = {}) {
|
||||||
|
this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
||||||
|
this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
|
||||||
|
this.pendingRequests = new Map();
|
||||||
|
this.sessionEmitters = new Map();
|
||||||
|
this.worker = null;
|
||||||
|
this.stderrTail = '';
|
||||||
|
this.inFlightRequests = 0;
|
||||||
|
this.idleTimer = null;
|
||||||
|
this.intentionalCloses = new WeakSet();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synthesize speech in the worker. Returns WAV bytes.
|
||||||
|
* @param {{ modelsDir: string, modelId: string, text: string, speakerId?: number, speed?: number }} params
|
||||||
|
* @returns {Promise<{ audio: Buffer, format: string }>}
|
||||||
|
*/
|
||||||
|
async synthesizeSpeech(params) {
|
||||||
|
// Long texts on slow hardware can exceed the default request timeout.
|
||||||
|
const result = await this.sendRequest(
|
||||||
|
{ type: 'tts.synthesize', ...params },
|
||||||
|
{ timeoutMs: 120000 },
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
audio: Buffer.isBuffer(result.audio) ? result.audio : Buffer.from(result.audio),
|
||||||
|
format: result.format || 'audio/wav',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a streaming STT session in the worker.
|
||||||
|
* @param {{ modelsDir: string, modelId: string }} params
|
||||||
|
* @param {EventEmitter} emitter receives 'committed' | 'transcript' | 'error'
|
||||||
|
* @returns {Promise<{ sessionId: string, requiredSampleRate: number }>}
|
||||||
|
*/
|
||||||
|
async createSession({ modelsDir, modelId }, emitter) {
|
||||||
|
const sessionId = randomUUID();
|
||||||
|
this.sessionEmitters.set(sessionId, emitter);
|
||||||
|
try {
|
||||||
|
const result = await this.sendRequest({
|
||||||
|
type: 'session.create',
|
||||||
|
sessionId,
|
||||||
|
modelsDir,
|
||||||
|
modelId,
|
||||||
|
});
|
||||||
|
return { sessionId, requiredSampleRate: result?.requiredSampleRate ?? DEFAULT_LOCAL_SAMPLE_RATE };
|
||||||
|
} catch (err) {
|
||||||
|
this.sessionEmitters.delete(sessionId);
|
||||||
|
this.scheduleIdleShutdownIfReady();
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
appendSessionAudio(sessionId, audio) {
|
||||||
|
void this.sendRequest({ type: 'session.append', sessionId, audio }).catch((err) => {
|
||||||
|
this.emitSessionError(sessionId, err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
commitSession(sessionId) {
|
||||||
|
void this.sendRequest({ type: 'session.commit', sessionId }).catch((err) => {
|
||||||
|
this.emitSessionError(sessionId, err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
clearSession(sessionId) {
|
||||||
|
void this.sendRequest({ type: 'session.clear', sessionId }).catch((err) => {
|
||||||
|
this.emitSessionError(sessionId, err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
closeSession(sessionId) {
|
||||||
|
this.sessionEmitters.delete(sessionId);
|
||||||
|
void this.sendRequest({ type: 'session.close', sessionId }).catch(() => {
|
||||||
|
// Closing is best-effort; the parent already dropped the session.
|
||||||
|
});
|
||||||
|
this.scheduleIdleShutdownIfReady();
|
||||||
|
}
|
||||||
|
|
||||||
|
shutdown() {
|
||||||
|
this.clearIdleTimer();
|
||||||
|
this.rejectAllPending(new Error('Dictation worker shut down'));
|
||||||
|
this.sessionEmitters.clear();
|
||||||
|
const worker = this.worker;
|
||||||
|
this.worker = null;
|
||||||
|
if (worker && !worker.killed) {
|
||||||
|
this.intentionalCloses.add(worker);
|
||||||
|
try {
|
||||||
|
worker.disconnect();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
worker.kill();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sendRequest(input, options = {}) {
|
||||||
|
const worker = this.ensureWorker();
|
||||||
|
const requestId = randomUUID();
|
||||||
|
const message = { ...input, requestId };
|
||||||
|
this.inFlightRequests += 1;
|
||||||
|
this.clearIdleTimer();
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
this.pendingRequests.delete(requestId);
|
||||||
|
this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
|
||||||
|
this.scheduleIdleShutdownIfReady();
|
||||||
|
reject(new Error(`Dictation worker request timed out: ${input.type}`));
|
||||||
|
}, options.timeoutMs ?? this.requestTimeoutMs);
|
||||||
|
|
||||||
|
this.pendingRequests.set(requestId, { resolve, reject, timeout });
|
||||||
|
|
||||||
|
worker.send(message, (error) => {
|
||||||
|
if (!error) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const pending = this.pendingRequests.get(requestId);
|
||||||
|
if (!pending) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
this.pendingRequests.delete(requestId);
|
||||||
|
this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
|
||||||
|
this.scheduleIdleShutdownIfReady();
|
||||||
|
pending.reject(error);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
ensureWorker() {
|
||||||
|
if (this.worker && !this.worker.killed && this.worker.connected) {
|
||||||
|
return this.worker;
|
||||||
|
}
|
||||||
|
const worker = forkDictationWorker();
|
||||||
|
this.worker = worker;
|
||||||
|
this.stderrTail = '';
|
||||||
|
worker.stderr?.on('data', (chunk) => {
|
||||||
|
const text = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk);
|
||||||
|
this.stderrTail = (this.stderrTail + text).slice(-STDERR_TAIL_MAX_CHARS);
|
||||||
|
});
|
||||||
|
worker.on('message', (message) => this.handleWorkerMessage(message));
|
||||||
|
worker.on('close', (code, signal) => this.handleWorkerExit(worker, code, signal));
|
||||||
|
return worker;
|
||||||
|
}
|
||||||
|
|
||||||
|
handleWorkerMessage(message) {
|
||||||
|
if (message?.type === 'response') {
|
||||||
|
const pending = this.pendingRequests.get(message.requestId);
|
||||||
|
if (!pending) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
this.pendingRequests.delete(message.requestId);
|
||||||
|
this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
|
||||||
|
this.scheduleIdleShutdownIfReady();
|
||||||
|
if (message.ok) {
|
||||||
|
pending.resolve(message.result);
|
||||||
|
} else {
|
||||||
|
pending.reject(new Error(message.error || 'Dictation worker request failed'));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const emitter = this.sessionEmitters.get(message?.sessionId);
|
||||||
|
if (!emitter) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
switch (message.type) {
|
||||||
|
case 'session.committed':
|
||||||
|
emitter.emit('committed', message.payload);
|
||||||
|
return;
|
||||||
|
case 'session.transcript':
|
||||||
|
emitter.emit('transcript', message.payload);
|
||||||
|
return;
|
||||||
|
case 'session.error':
|
||||||
|
emitter.emit('error', new Error(message.error));
|
||||||
|
return;
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handleWorkerExit(worker, code, signal) {
|
||||||
|
const wasCurrentWorker = this.worker === worker;
|
||||||
|
const wasIntentional = this.intentionalCloses.has(worker);
|
||||||
|
this.intentionalCloses.delete(worker);
|
||||||
|
if (!wasCurrentWorker || wasIntentional) {
|
||||||
|
if (wasCurrentWorker) {
|
||||||
|
this.worker = null;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stderr = this.stderrTail.trim();
|
||||||
|
const error = new Error(
|
||||||
|
`Dictation worker exited (code ${code ?? 'null'}${signal ? `, signal ${signal}` : ''}).` +
|
||||||
|
(stderr ? ` Last stderr: ${stderr.slice(-500)}` : ''),
|
||||||
|
);
|
||||||
|
|
||||||
|
this.worker = null;
|
||||||
|
this.clearIdleTimer();
|
||||||
|
this.rejectAllPending(error);
|
||||||
|
for (const emitter of this.sessionEmitters.values()) {
|
||||||
|
if (emitter.listenerCount('error') > 0) {
|
||||||
|
emitter.emit('error', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.sessionEmitters.clear();
|
||||||
|
this.inFlightRequests = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
rejectAllPending(error) {
|
||||||
|
for (const [requestId, pending] of this.pendingRequests) {
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
pending.reject(error);
|
||||||
|
this.pendingRequests.delete(requestId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
emitSessionError(sessionId, error) {
|
||||||
|
const emitter = this.sessionEmitters.get(sessionId);
|
||||||
|
if (emitter && emitter.listenerCount('error') > 0) {
|
||||||
|
emitter.emit('error', error instanceof Error ? error : new Error(String(error)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleIdleShutdownIfReady() {
|
||||||
|
if (!this.worker || this.inFlightRequests > 0 || this.sessionEmitters.size > 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.clearIdleTimer();
|
||||||
|
this.idleTimer = setTimeout(() => {
|
||||||
|
if (this.inFlightRequests === 0 && this.sessionEmitters.size === 0) {
|
||||||
|
this.shutdown();
|
||||||
|
}
|
||||||
|
}, this.idleTtlMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
clearIdleTimer() {
|
||||||
|
if (this.idleTimer) {
|
||||||
|
clearTimeout(this.idleTimer);
|
||||||
|
this.idleTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* StreamingTranscriptionSession backed by the worker process.
|
||||||
|
* Matches the session contract consumed by DictationStreamManager.
|
||||||
|
*/
|
||||||
|
export class WorkerBackedTranscriptionSession extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* @param {DictationWorkerClient} client
|
||||||
|
* @param {{ modelsDir: string, modelId: string }} modelConfig
|
||||||
|
*/
|
||||||
|
constructor(client, modelConfig) {
|
||||||
|
super();
|
||||||
|
this.client = client;
|
||||||
|
this.modelConfig = modelConfig;
|
||||||
|
this.requiredSampleRate = DEFAULT_LOCAL_SAMPLE_RATE;
|
||||||
|
this.connectedSessionId = null;
|
||||||
|
this.connecting = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async connect() {
|
||||||
|
if (this.connectedSessionId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!this.connecting) {
|
||||||
|
this.connecting = (async () => {
|
||||||
|
try {
|
||||||
|
const result = await this.client.createSession(this.modelConfig, this);
|
||||||
|
this.connectedSessionId = result.sessionId;
|
||||||
|
this.requiredSampleRate = result.requiredSampleRate;
|
||||||
|
} finally {
|
||||||
|
this.connecting = null;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
await this.connecting;
|
||||||
|
}
|
||||||
|
|
||||||
|
appendPcm16(pcm16le) {
|
||||||
|
if (!this.connectedSessionId) {
|
||||||
|
this.emit('error', new Error('Local STT session not connected'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.client.appendSessionAudio(this.connectedSessionId, pcm16le);
|
||||||
|
}
|
||||||
|
|
||||||
|
commit() {
|
||||||
|
if (!this.connectedSessionId) {
|
||||||
|
this.emit('error', new Error('Local STT session not connected'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.client.commitSession(this.connectedSessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
if (this.connectedSessionId) {
|
||||||
|
this.client.clearSession(this.connectedSessionId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
close() {
|
||||||
|
const sessionId = this.connectedSessionId;
|
||||||
|
this.connectedSessionId = null;
|
||||||
|
if (sessionId) {
|
||||||
|
this.client.closeSession(sessionId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
/**
|
||||||
|
* Dictation local-speech worker process.
|
||||||
|
*
|
||||||
|
* Hosts the sherpa-onnx native inference (Parakeet STT) in a separate process
|
||||||
|
* so ONNX decoding never blocks the main OpenChamber server. Communicates
|
||||||
|
* with the parent over child_process IPC (advanced serialization, so Buffers
|
||||||
|
* survive the trip as Uint8Array).
|
||||||
|
*
|
||||||
|
* Request/response protocol (parent -> worker):
|
||||||
|
* { type: 'session.create', requestId, sessionId, modelsDir, modelId }
|
||||||
|
* { type: 'session.append', requestId, sessionId, audio }
|
||||||
|
* { type: 'session.commit' | 'session.clear' | 'session.close', requestId, sessionId }
|
||||||
|
* Worker -> parent:
|
||||||
|
* { type: 'response', requestId, ok, result?, error? }
|
||||||
|
* { type: 'session.committed' | 'session.transcript' | 'session.error', sessionId, ... }
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
SherpaOfflineRecognizerEngine,
|
||||||
|
SherpaRealtimeTranscriptionSession,
|
||||||
|
} from './sherpa-recognizer.js';
|
||||||
|
import { SherpaTtsEngine } from './sherpa-tts.js';
|
||||||
|
import { getLocalSttModelDir, getLocalSttModelSpec } from './model-catalog.js';
|
||||||
|
import { pcm16ToWav } from '../audio.js';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
process.title = 'OpenChamber Dictation';
|
||||||
|
|
||||||
|
const engines = new Map();
|
||||||
|
const ttsEngines = new Map();
|
||||||
|
const sessions = new Map();
|
||||||
|
let ipcClosing = false;
|
||||||
|
|
||||||
|
function sendToParent(message) {
|
||||||
|
if (ipcClosing || !process.connected || !process.send) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
process.send(message, (error) => {
|
||||||
|
if (error) {
|
||||||
|
ipcClosing = true;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
ipcClosing = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendOk(requestId, result) {
|
||||||
|
sendToParent({ type: 'response', requestId, ok: true, ...(result !== undefined ? { result } : {}) });
|
||||||
|
}
|
||||||
|
|
||||||
|
function getEngine(modelsDir, modelId) {
|
||||||
|
const key = `${modelsDir}:${modelId}`;
|
||||||
|
const existing = engines.get(key);
|
||||||
|
if (existing) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
const modelDir = getLocalSttModelDir(modelsDir, modelId);
|
||||||
|
const spec = getLocalSttModelSpec(modelId);
|
||||||
|
const created = new SherpaOfflineRecognizerEngine({
|
||||||
|
type: spec.type,
|
||||||
|
encoder: path.join(modelDir, spec.files.encoder),
|
||||||
|
decoder: path.join(modelDir, spec.files.decoder),
|
||||||
|
...(spec.files.joiner ? { joiner: path.join(modelDir, spec.files.joiner) } : {}),
|
||||||
|
tokens: path.join(modelDir, spec.files.tokens),
|
||||||
|
numThreads: 2,
|
||||||
|
});
|
||||||
|
engines.set(key, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanupSession(sessionId) {
|
||||||
|
const session = sessions.get(sessionId);
|
||||||
|
sessions.delete(sessionId);
|
||||||
|
try {
|
||||||
|
session?.close();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function toBuffer(audio) {
|
||||||
|
if (Buffer.isBuffer(audio)) {
|
||||||
|
return audio;
|
||||||
|
}
|
||||||
|
if (audio instanceof Uint8Array) {
|
||||||
|
return Buffer.from(audio.buffer, audio.byteOffset, audio.byteLength);
|
||||||
|
}
|
||||||
|
if (audio && typeof audio === 'object' && audio.type === 'Buffer' && Array.isArray(audio.data)) {
|
||||||
|
return Buffer.from(audio.data);
|
||||||
|
}
|
||||||
|
throw new Error('Unsupported audio payload in dictation worker');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTtsEngine(modelsDir, modelId) {
|
||||||
|
const key = `${modelsDir}:${modelId}`;
|
||||||
|
const existing = ttsEngines.get(key);
|
||||||
|
if (existing) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
const spec = getLocalSttModelSpec(modelId);
|
||||||
|
const created = new SherpaTtsEngine({
|
||||||
|
modelDir: getLocalSttModelDir(modelsDir, modelId),
|
||||||
|
files: spec.files,
|
||||||
|
numThreads: 2,
|
||||||
|
});
|
||||||
|
ttsEngines.set(key, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleRequest(message) {
|
||||||
|
switch (message.type) {
|
||||||
|
case 'tts.synthesize': {
|
||||||
|
const engine = getTtsEngine(message.modelsDir, message.modelId);
|
||||||
|
const { pcm16, sampleRate } = engine.synthesize(message.text, {
|
||||||
|
speakerId: message.speakerId,
|
||||||
|
speed: message.speed,
|
||||||
|
});
|
||||||
|
sendOk(message.requestId, {
|
||||||
|
audio: pcm16ToWav(pcm16, sampleRate),
|
||||||
|
format: 'audio/wav',
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'session.create': {
|
||||||
|
cleanupSession(message.sessionId);
|
||||||
|
const engine = getEngine(message.modelsDir, message.modelId);
|
||||||
|
const session = new SherpaRealtimeTranscriptionSession({ engine });
|
||||||
|
session.on('committed', (payload) => {
|
||||||
|
sendToParent({ type: 'session.committed', sessionId: message.sessionId, payload });
|
||||||
|
});
|
||||||
|
session.on('transcript', (payload) => {
|
||||||
|
sendToParent({ type: 'session.transcript', sessionId: message.sessionId, payload });
|
||||||
|
});
|
||||||
|
session.on('error', (err) => {
|
||||||
|
sendToParent({
|
||||||
|
type: 'session.error',
|
||||||
|
sessionId: message.sessionId,
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await session.connect();
|
||||||
|
sessions.set(message.sessionId, session);
|
||||||
|
sendOk(message.requestId, { requiredSampleRate: session.requiredSampleRate });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'session.append': {
|
||||||
|
sessions.get(message.sessionId)?.appendPcm16(toBuffer(message.audio));
|
||||||
|
sendOk(message.requestId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'session.commit': {
|
||||||
|
sessions.get(message.sessionId)?.commit();
|
||||||
|
sendOk(message.requestId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'session.clear': {
|
||||||
|
sessions.get(message.sessionId)?.clear();
|
||||||
|
sendOk(message.requestId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'session.close': {
|
||||||
|
cleanupSession(message.sessionId);
|
||||||
|
sendOk(message.requestId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
default: {
|
||||||
|
throw new Error(`Unknown dictation worker request: ${message?.type}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on('message', (message) => {
|
||||||
|
void handleRequest(message).catch((error) => {
|
||||||
|
sendToParent({
|
||||||
|
type: 'response',
|
||||||
|
requestId: message?.requestId,
|
||||||
|
ok: false,
|
||||||
|
error: error instanceof Error ? error.message : 'Dictation worker request failed',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
process.once('disconnect', () => {
|
||||||
|
ipcClosing = true;
|
||||||
|
for (const sessionId of Array.from(sessions.keys())) {
|
||||||
|
cleanupSession(sessionId);
|
||||||
|
}
|
||||||
|
for (const engine of engines.values()) {
|
||||||
|
engine.free();
|
||||||
|
}
|
||||||
|
for (const tts of ttsEngines.values()) {
|
||||||
|
tts.free();
|
||||||
|
}
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
/**
|
||||||
|
* Pseudo-streaming transcription session for OpenAI-compatible Whisper
|
||||||
|
* endpoints (faster-whisper, whisper.cpp, OpenAI, ...).
|
||||||
|
*
|
||||||
|
* The Whisper HTTP API cannot stream, so audio is buffered per segment and
|
||||||
|
* transcribed on commit(). Live partials therefore only advance at segment
|
||||||
|
* boundaries (the DictationStreamManager auto-commits every ~15s of speech).
|
||||||
|
*
|
||||||
|
* Implements the StreamingTranscriptionSession contract used by
|
||||||
|
* DictationStreamManager.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { EventEmitter } from 'events';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
|
||||||
|
import { transcribeAudio } from '../tts/stt.js';
|
||||||
|
import { pcm16ToWav } from './audio.js';
|
||||||
|
|
||||||
|
const OPENAI_COMPATIBLE_SAMPLE_RATE = 16000;
|
||||||
|
|
||||||
|
export class OpenAICompatibleTranscriptionSession extends EventEmitter {
|
||||||
|
/**
|
||||||
|
* @param {{ baseURL: string, model: string, apiKey?: string, language?: string, prompt?: string }} config
|
||||||
|
*/
|
||||||
|
constructor(config) {
|
||||||
|
super();
|
||||||
|
this.config = config;
|
||||||
|
this.requiredSampleRate = OPENAI_COMPATIBLE_SAMPLE_RATE;
|
||||||
|
this.connected = false;
|
||||||
|
this.segmentId = randomUUID();
|
||||||
|
this.previousSegmentId = null;
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async connect() {
|
||||||
|
if (!this.config.baseURL) {
|
||||||
|
throw new Error('Custom STT server URL is not configured');
|
||||||
|
}
|
||||||
|
if (!this.config.model) {
|
||||||
|
throw new Error('STT model is not configured');
|
||||||
|
}
|
||||||
|
this.connected = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
appendPcm16(chunk) {
|
||||||
|
if (!this.connected) {
|
||||||
|
this.emit('error', new Error('STT session not connected'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
|
||||||
|
}
|
||||||
|
|
||||||
|
commit() {
|
||||||
|
if (!this.connected) {
|
||||||
|
this.emit('error', new Error('STT session not connected'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const committedId = this.segmentId;
|
||||||
|
const previousSegmentId = this.previousSegmentId;
|
||||||
|
const committedPcm16 = this.pcm16;
|
||||||
|
this.previousSegmentId = committedId;
|
||||||
|
this.segmentId = randomUUID();
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
this.emit('committed', { segmentId: committedId, previousSegmentId });
|
||||||
|
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
const wav = pcm16ToWav(committedPcm16, OPENAI_COMPATIBLE_SAMPLE_RATE);
|
||||||
|
const text = await transcribeAudio({
|
||||||
|
audioBuffer: wav,
|
||||||
|
mimeType: 'audio/wav',
|
||||||
|
model: this.config.model,
|
||||||
|
baseURL: this.config.baseURL,
|
||||||
|
apiKey: this.config.apiKey,
|
||||||
|
language: this.config.language,
|
||||||
|
});
|
||||||
|
this.emit('transcript', {
|
||||||
|
segmentId: committedId,
|
||||||
|
transcript: (text ?? '').trim(),
|
||||||
|
isFinal: true,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
this.segmentId = randomUUID();
|
||||||
|
}
|
||||||
|
|
||||||
|
close() {
|
||||||
|
this.connected = false;
|
||||||
|
this.pcm16 = Buffer.alloc(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
/**
|
||||||
|
* Dictation runtime: registers the streaming dictation WebSocket endpoint and
|
||||||
|
* the HTTP status/model routes.
|
||||||
|
*
|
||||||
|
* WebSocket protocol (JSON text frames) on /api/dictation/ws:
|
||||||
|
* client -> server:
|
||||||
|
* { type: 'start', dictationId, format, options? }
|
||||||
|
* options: { provider?, language?, localModel?, openaiCompatible? }
|
||||||
|
* { type: 'chunk', dictationId, seq, audio } // audio: base64 PCM16LE
|
||||||
|
* { type: 'finish', dictationId, finalSeq }
|
||||||
|
* { type: 'cancel', dictationId }
|
||||||
|
* { type: 'ping' }
|
||||||
|
* server -> client:
|
||||||
|
* { type: 'ready' }
|
||||||
|
* { type: 'ack', dictationId, ackSeq }
|
||||||
|
* { type: 'partial', dictationId, text }
|
||||||
|
* { type: 'finish_accepted', dictationId, timeoutMs }
|
||||||
|
* { type: 'final', dictationId, text }
|
||||||
|
* { type: 'error', dictationId, error, retryable, reasonCode? }
|
||||||
|
* { type: 'pong' }
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { WebSocketServer } from 'ws';
|
||||||
|
|
||||||
|
import { DictationStreamManager } from './stream-manager.js';
|
||||||
|
import { createDictationService } from './service.js';
|
||||||
|
|
||||||
|
const DICTATION_WS_PATH = '/api/dictation/ws';
|
||||||
|
|
||||||
|
const DICTATION_WS_MAX_PAYLOAD_BYTES = 512 * 1024;
|
||||||
|
const DICTATION_WS_HEARTBEAT_INTERVAL_MS = 30000;
|
||||||
|
|
||||||
|
const parseRequestPathname = (url) => {
|
||||||
|
try {
|
||||||
|
return new URL(url, 'http://localhost').pathname;
|
||||||
|
} catch {
|
||||||
|
return typeof url === 'string' ? url.split('?')[0] : '';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export function createDictationRuntime({
|
||||||
|
app,
|
||||||
|
server,
|
||||||
|
express,
|
||||||
|
uiAuthController,
|
||||||
|
isRequestOriginAllowed,
|
||||||
|
rejectWebSocketUpgrade,
|
||||||
|
modelsDir,
|
||||||
|
}) {
|
||||||
|
const service = createDictationService({ modelsDir });
|
||||||
|
|
||||||
|
// Local text-to-speech (Kokoro in the dictation worker). Returns WAV bytes;
|
||||||
|
// 503 with a reason code while the model is still downloading.
|
||||||
|
app.post('/api/dictation/tts/speak', express.json({ limit: '1mb' }), async (req, res) => {
|
||||||
|
try {
|
||||||
|
const text = typeof req.body?.text === 'string' ? req.body.text.trim() : '';
|
||||||
|
if (!text) {
|
||||||
|
res.status(400).json({ error: 'Text is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const result = await service.synthesizeSpeech({
|
||||||
|
text,
|
||||||
|
model: typeof req.body?.model === 'string' ? req.body.model : undefined,
|
||||||
|
speakerId: Number.isInteger(req.body?.speakerId) ? req.body.speakerId : undefined,
|
||||||
|
speed: typeof req.body?.speed === 'number' ? req.body.speed : undefined,
|
||||||
|
});
|
||||||
|
if (result.error) {
|
||||||
|
res.status(503).json({
|
||||||
|
error: result.error,
|
||||||
|
retryable: result.retryable !== false,
|
||||||
|
...(result.reasonCode ? { reasonCode: result.reasonCode } : {}),
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.setHeader('Content-Type', result.format || 'audio/wav');
|
||||||
|
res.send(result.audio);
|
||||||
|
} catch (error) {
|
||||||
|
res.status(500).json({ error: error?.message || 'Failed to synthesize speech' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get('/api/dictation/status', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const provider = typeof req.query.provider === 'string' ? req.query.provider : undefined;
|
||||||
|
const localModel = typeof req.query.localModel === 'string' ? req.query.localModel : undefined;
|
||||||
|
const status = await service.getStatus({ provider, localModel });
|
||||||
|
res.json(status);
|
||||||
|
} catch (error) {
|
||||||
|
res.status(500).json({ error: error?.message || 'Failed to read dictation status' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post('/api/dictation/models/:modelId/download', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const result = await service.requestModelDownload(req.params.modelId);
|
||||||
|
if (!result.ok) {
|
||||||
|
res.status(400).json({ error: result.error });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.json(result);
|
||||||
|
} catch (error) {
|
||||||
|
res.status(500).json({ error: error?.message || 'Failed to start model download' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.delete('/api/dictation/models/:modelId', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const result = await service.deleteModel(req.params.modelId);
|
||||||
|
if (!result.ok) {
|
||||||
|
res.status(400).json({ error: result.error });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.json(result);
|
||||||
|
} catch (error) {
|
||||||
|
res.status(500).json({ error: error?.message || 'Failed to delete model' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const wsServer = new WebSocketServer({
|
||||||
|
noServer: true,
|
||||||
|
maxPayload: DICTATION_WS_MAX_PAYLOAD_BYTES,
|
||||||
|
});
|
||||||
|
|
||||||
|
wsServer.on('connection', (socket) => {
|
||||||
|
const send = (msg) => {
|
||||||
|
if (socket.readyState !== 1) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
socket.send(JSON.stringify(msg));
|
||||||
|
} catch {
|
||||||
|
// socket is going away; the manager cleanup on close handles state
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const manager = new DictationStreamManager({
|
||||||
|
emit: ({ type, payload }) => send({ type, ...payload }),
|
||||||
|
createSttSession: (options) => service.createSttSession(options),
|
||||||
|
});
|
||||||
|
|
||||||
|
send({ type: 'ready' });
|
||||||
|
|
||||||
|
const heartbeatInterval = setInterval(() => {
|
||||||
|
if (socket.readyState !== 1) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
socket.ping();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}, DICTATION_WS_HEARTBEAT_INTERVAL_MS);
|
||||||
|
|
||||||
|
socket.on('message', (raw, isBinary) => {
|
||||||
|
if (isBinary) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let message;
|
||||||
|
try {
|
||||||
|
message = JSON.parse(raw.toString('utf8'));
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!message || typeof message !== 'object') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (message.type) {
|
||||||
|
case 'start': {
|
||||||
|
if (typeof message.dictationId !== 'string' || typeof message.format !== 'string') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const options =
|
||||||
|
message.options && typeof message.options === 'object' ? message.options : {};
|
||||||
|
void manager.handleStart(message.dictationId, message.format, options);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'chunk': {
|
||||||
|
if (
|
||||||
|
typeof message.dictationId !== 'string' ||
|
||||||
|
typeof message.seq !== 'number' ||
|
||||||
|
typeof message.audio !== 'string'
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
manager.handleChunk({
|
||||||
|
dictationId: message.dictationId,
|
||||||
|
seq: message.seq,
|
||||||
|
audioBase64: message.audio,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'finish': {
|
||||||
|
if (typeof message.dictationId !== 'string' || typeof message.finalSeq !== 'number') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
manager.handleFinish(message.dictationId, message.finalSeq);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'cancel': {
|
||||||
|
if (typeof message.dictationId !== 'string') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
manager.handleCancel(message.dictationId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case 'ping': {
|
||||||
|
send({ type: 'pong' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('close', () => {
|
||||||
|
clearInterval(heartbeatInterval);
|
||||||
|
manager.cleanupAll();
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('error', () => {
|
||||||
|
// 'close' follows and performs cleanup.
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const upgradeHandler = (req, socket, head) => {
|
||||||
|
const pathname = parseRequestPathname(req.url);
|
||||||
|
if (pathname !== DICTATION_WS_PATH) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleUpgrade = async () => {
|
||||||
|
try {
|
||||||
|
if (uiAuthController?.enabled) {
|
||||||
|
const sessionToken = await uiAuthController?.ensureSessionToken?.(req, null);
|
||||||
|
if (!sessionToken) {
|
||||||
|
rejectWebSocketUpgrade(socket, 401, 'UI authentication required');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const originAllowed = await isRequestOriginAllowed(req);
|
||||||
|
if (!originAllowed) {
|
||||||
|
rejectWebSocketUpgrade(socket, 403, 'Invalid origin');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wsServer.handleUpgrade(req, socket, head, (ws) => {
|
||||||
|
wsServer.emit('connection', ws, req);
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
rejectWebSocketUpgrade(socket, 500, 'Upgrade failed');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
void handleUpgrade();
|
||||||
|
};
|
||||||
|
|
||||||
|
server.on('upgrade', upgradeHandler);
|
||||||
|
|
||||||
|
const stop = () => {
|
||||||
|
server.off('upgrade', upgradeHandler);
|
||||||
|
for (const client of wsServer.clients) {
|
||||||
|
try {
|
||||||
|
client.close(1001, 'server shutting down');
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
wsServer.close();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
service.shutdown();
|
||||||
|
};
|
||||||
|
|
||||||
|
return { stop };
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
/**
|
||||||
|
* Dictation service: resolves STT providers, tracks local model download
|
||||||
|
* state, and exposes a readiness snapshot for the status route.
|
||||||
|
*
|
||||||
|
* Providers:
|
||||||
|
* - 'local' (default): sherpa-onnx Parakeet running in a worker process.
|
||||||
|
* Models auto-download in the background on first use.
|
||||||
|
* - 'openai-compatible': any OpenAI-compatible /v1/audio/transcriptions
|
||||||
|
* endpoint (faster-whisper, whisper.cpp, OpenAI).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { rm } from 'fs/promises';
|
||||||
|
|
||||||
|
import { DictationWorkerClient, WorkerBackedTranscriptionSession } from './local/worker-client.js';
|
||||||
|
import { OpenAICompatibleTranscriptionSession } from './openai-compatible-session.js';
|
||||||
|
import {
|
||||||
|
DEFAULT_LOCAL_STT_MODEL,
|
||||||
|
DEFAULT_LOCAL_TTS_MODEL,
|
||||||
|
LOCAL_STT_MODEL_CATALOG,
|
||||||
|
LOCAL_STT_MODEL_IDS,
|
||||||
|
LOCAL_TTS_MODEL_CATALOG,
|
||||||
|
LOCAL_TTS_MODEL_IDS,
|
||||||
|
getLocalSttModelDir,
|
||||||
|
isLocalModelId,
|
||||||
|
isLocalSttModelId,
|
||||||
|
isLocalTtsModelId,
|
||||||
|
} from './local/model-catalog.js';
|
||||||
|
import { ensureLocalSttModel, isLocalSttModelInstalled } from './local/model-downloader.js';
|
||||||
|
|
||||||
|
export function createDictationService({ modelsDir }) {
|
||||||
|
const workerClient = new DictationWorkerClient();
|
||||||
|
/** modelId -> 'downloading' | 'error' */
|
||||||
|
const downloadStates = new Map();
|
||||||
|
/** modelId -> last download error message */
|
||||||
|
const downloadErrors = new Map();
|
||||||
|
/** modelId -> in-flight ensure promise */
|
||||||
|
const downloadPromises = new Map();
|
||||||
|
/** modelId -> 0..100 download percent (null while size unknown) */
|
||||||
|
const downloadProgress = new Map();
|
||||||
|
|
||||||
|
const startModelDownload = (modelId) => {
|
||||||
|
const existing = downloadPromises.get(modelId);
|
||||||
|
if (existing) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
downloadStates.set(modelId, 'downloading');
|
||||||
|
downloadErrors.delete(modelId);
|
||||||
|
downloadProgress.set(modelId, 0);
|
||||||
|
const promise = ensureLocalSttModel({
|
||||||
|
modelsDir,
|
||||||
|
modelId,
|
||||||
|
onProgress: (downloadedBytes, totalBytes) => {
|
||||||
|
downloadProgress.set(
|
||||||
|
modelId,
|
||||||
|
totalBytes ? Math.min(100, Math.round((downloadedBytes / totalBytes) * 100)) : null,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.then(() => {
|
||||||
|
downloadStates.delete(modelId);
|
||||||
|
downloadPromises.delete(modelId);
|
||||||
|
downloadProgress.delete(modelId);
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
downloadStates.set(modelId, 'error');
|
||||||
|
downloadErrors.set(modelId, error?.message || String(error));
|
||||||
|
downloadPromises.delete(modelId);
|
||||||
|
downloadProgress.delete(modelId);
|
||||||
|
});
|
||||||
|
downloadPromises.set(modelId, promise);
|
||||||
|
return promise;
|
||||||
|
};
|
||||||
|
|
||||||
|
const resolveLocalModelId = (requested) => {
|
||||||
|
return isLocalSttModelId(requested) ? requested : DEFAULT_LOCAL_STT_MODEL;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a connected StreamingTranscriptionSession for one dictation.
|
||||||
|
* Returns { session } on success or { error, retryable, reasonCode } when
|
||||||
|
* the provider is not ready.
|
||||||
|
*
|
||||||
|
* @param {{ provider?: string, language?: string, localModel?: string,
|
||||||
|
* openaiCompatible?: { baseUrl?: string, model?: string, apiKey?: string } }} options
|
||||||
|
*/
|
||||||
|
const createSttSession = async (options = {}) => {
|
||||||
|
const provider = options.provider === 'openai-compatible' ? 'openai-compatible' : 'local';
|
||||||
|
|
||||||
|
if (provider === 'openai-compatible') {
|
||||||
|
const config = options.openaiCompatible || {};
|
||||||
|
const session = new OpenAICompatibleTranscriptionSession({
|
||||||
|
baseURL: config.baseUrl,
|
||||||
|
model: config.model,
|
||||||
|
apiKey: config.apiKey || undefined,
|
||||||
|
language: options.language || undefined,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await session.connect();
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
error: error?.message || String(error),
|
||||||
|
retryable: false,
|
||||||
|
reasonCode: 'stt_not_configured',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { session };
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelId = resolveLocalModelId(options.localModel);
|
||||||
|
const installed = await isLocalSttModelInstalled(modelsDir, modelId);
|
||||||
|
if (!installed) {
|
||||||
|
const state = downloadStates.get(modelId);
|
||||||
|
if (state === 'error') {
|
||||||
|
const message = downloadErrors.get(modelId) || 'Model download failed';
|
||||||
|
// Allow a retry on the next attempt.
|
||||||
|
downloadStates.delete(modelId);
|
||||||
|
return {
|
||||||
|
error: `Failed to download dictation model: ${message}`,
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'model_download_failed',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
void startModelDownload(modelId);
|
||||||
|
return {
|
||||||
|
error: 'Dictation model is downloading',
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'model_download_in_progress',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const session = new WorkerBackedTranscriptionSession(workerClient, { modelsDir, modelId });
|
||||||
|
try {
|
||||||
|
await session.connect();
|
||||||
|
} catch (error) {
|
||||||
|
const message = error?.message || String(error);
|
||||||
|
// A model that passes the file-presence check but fails to load is
|
||||||
|
// corrupt on disk (e.g. truncated by an interrupted extraction). Remove
|
||||||
|
// it so the next attempt re-downloads instead of crashing forever.
|
||||||
|
if (/Load model|Protobuf parsing failed/i.test(message)) {
|
||||||
|
await rm(getLocalSttModelDir(modelsDir, modelId), { recursive: true, force: true })
|
||||||
|
.catch(() => undefined);
|
||||||
|
return {
|
||||||
|
error: 'Dictation model files were corrupt and have been removed; retry to re-download',
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'model_corrupt',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
error: message,
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'stt_unavailable',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { session };
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Readiness snapshot for the status route and UI gating.
|
||||||
|
* @param {{ provider?: string, localModel?: string }} [options]
|
||||||
|
*/
|
||||||
|
const getStatus = async (options = {}) => {
|
||||||
|
const provider = options.provider === 'openai-compatible' ? 'openai-compatible' : 'local';
|
||||||
|
const modelId = resolveLocalModelId(options.localModel);
|
||||||
|
|
||||||
|
const describeModel = async (id, catalog) => ({
|
||||||
|
id,
|
||||||
|
description: catalog[id].description,
|
||||||
|
installed: await isLocalSttModelInstalled(modelsDir, id),
|
||||||
|
downloading: downloadStates.get(id) === 'downloading',
|
||||||
|
downloadProgress: downloadProgress.get(id) ?? null,
|
||||||
|
downloadError: downloadErrors.get(id) || null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const models = await Promise.all(
|
||||||
|
LOCAL_STT_MODEL_IDS.map((id) => describeModel(id, LOCAL_STT_MODEL_CATALOG)),
|
||||||
|
);
|
||||||
|
const ttsModels = await Promise.all(
|
||||||
|
LOCAL_TTS_MODEL_IDS.map((id) => describeModel(id, LOCAL_TTS_MODEL_CATALOG)),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (provider === 'openai-compatible') {
|
||||||
|
return { provider, available: true, models, ttsModels };
|
||||||
|
}
|
||||||
|
|
||||||
|
const model = models.find((entry) => entry.id === modelId) || null;
|
||||||
|
if (model?.installed) {
|
||||||
|
return { provider, available: true, activeModel: modelId, models, ttsModels };
|
||||||
|
}
|
||||||
|
if (model?.downloading) {
|
||||||
|
return {
|
||||||
|
provider,
|
||||||
|
available: false,
|
||||||
|
reasonCode: 'model_download_in_progress',
|
||||||
|
activeModel: modelId,
|
||||||
|
models,
|
||||||
|
ttsModels,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (model?.downloadError) {
|
||||||
|
return {
|
||||||
|
provider,
|
||||||
|
available: false,
|
||||||
|
reasonCode: 'model_download_failed',
|
||||||
|
error: model.downloadError,
|
||||||
|
activeModel: modelId,
|
||||||
|
models,
|
||||||
|
ttsModels,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
provider,
|
||||||
|
available: false,
|
||||||
|
reasonCode: 'models_missing',
|
||||||
|
activeModel: modelId,
|
||||||
|
models,
|
||||||
|
ttsModels,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synthesize speech with the local TTS model. Returns WAV bytes, or a
|
||||||
|
* readiness error while the model is missing/downloading.
|
||||||
|
* @param {{ text: string, model?: string, speakerId?: number, speed?: number }} options
|
||||||
|
*/
|
||||||
|
const synthesizeSpeech = async ({ text, model, speakerId, speed }) => {
|
||||||
|
const modelId = isLocalTtsModelId(model) ? model : DEFAULT_LOCAL_TTS_MODEL;
|
||||||
|
const installed = await isLocalSttModelInstalled(modelsDir, modelId);
|
||||||
|
if (!installed) {
|
||||||
|
const state = downloadStates.get(modelId);
|
||||||
|
if (state === 'error') {
|
||||||
|
const message = downloadErrors.get(modelId) || 'Model download failed';
|
||||||
|
downloadStates.delete(modelId);
|
||||||
|
return {
|
||||||
|
error: `Failed to download TTS model: ${message}`,
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'model_download_failed',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
void startModelDownload(modelId);
|
||||||
|
return {
|
||||||
|
error: 'TTS model is downloading',
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'model_download_in_progress',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await workerClient.synthesizeSpeech({
|
||||||
|
modelsDir,
|
||||||
|
modelId,
|
||||||
|
text,
|
||||||
|
speakerId,
|
||||||
|
speed,
|
||||||
|
});
|
||||||
|
return { audio: result.audio, format: result.format };
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Kick off a background download for a model (used by the status route's
|
||||||
|
* download action so Settings can pre-download models).
|
||||||
|
*/
|
||||||
|
const requestModelDownload = async (modelId) => {
|
||||||
|
if (!isLocalModelId(modelId)) {
|
||||||
|
return { ok: false, error: 'Unknown model id' };
|
||||||
|
}
|
||||||
|
if (await isLocalSttModelInstalled(modelsDir, modelId)) {
|
||||||
|
return { ok: true, installed: true };
|
||||||
|
}
|
||||||
|
void startModelDownload(modelId);
|
||||||
|
return { ok: true, installed: false };
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an installed model from disk. A model that is mid-download cannot
|
||||||
|
* be deleted. An engine already loaded in the worker keeps its in-memory
|
||||||
|
* copy until the worker's idle shutdown; the files are simply re-downloaded
|
||||||
|
* on the next use if the model is selected again.
|
||||||
|
*/
|
||||||
|
const deleteModel = async (modelId) => {
|
||||||
|
if (!isLocalModelId(modelId)) {
|
||||||
|
return { ok: false, error: 'Unknown model id' };
|
||||||
|
}
|
||||||
|
if (downloadStates.get(modelId) === 'downloading') {
|
||||||
|
return { ok: false, error: 'Model is downloading' };
|
||||||
|
}
|
||||||
|
await rm(getLocalSttModelDir(modelsDir, modelId), { recursive: true, force: true });
|
||||||
|
downloadErrors.delete(modelId);
|
||||||
|
return { ok: true };
|
||||||
|
};
|
||||||
|
|
||||||
|
const shutdown = () => {
|
||||||
|
workerClient.shutdown();
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
createSttSession,
|
||||||
|
synthesizeSpeech,
|
||||||
|
getStatus,
|
||||||
|
requestModelDownload,
|
||||||
|
deleteModel,
|
||||||
|
shutdown,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,461 @@
|
|||||||
|
/**
|
||||||
|
* DictationStreamManager
|
||||||
|
*
|
||||||
|
* Server-authoritative streaming dictation state machine. One manager owns
|
||||||
|
* all dictation streams for a single WebSocket connection.
|
||||||
|
*
|
||||||
|
* Responsibilities:
|
||||||
|
* - Reorders inbound chunks by `seq` and acks the highest contiguous seq.
|
||||||
|
* - Resamples client PCM (16 kHz by default) to the provider's required rate.
|
||||||
|
* - Auto-commits a segment every `autoCommitSeconds` of audio, but clears
|
||||||
|
* silence-only segments instead of committing them.
|
||||||
|
* - Concatenates per-segment transcripts into live partials and emits the
|
||||||
|
* final text once every committed segment has a final transcript.
|
||||||
|
* - Applies an adaptive finalization timeout budget based on pending work.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Pcm16MonoResampler, parsePcmRateFromFormat, pcm16lePeakAbs } from './audio.js';
|
||||||
|
|
||||||
|
const DEFAULT_FINAL_TIMEOUT_MS = 10000;
|
||||||
|
const DEFAULT_AUTO_COMMIT_SECONDS = 15;
|
||||||
|
const FINAL_TIMEOUT_MAX_MS = 5 * 60 * 1000;
|
||||||
|
const FINAL_TIMEOUT_PER_PENDING_SEGMENT_MS = 15 * 1000;
|
||||||
|
const FINAL_TIMEOUT_PER_PENDING_AUDIO_SECOND_MS = 1500;
|
||||||
|
const FINAL_TIMEOUT_PER_MISSING_SEQ_MS = 250;
|
||||||
|
const SILENCE_PEAK_THRESHOLD = 300;
|
||||||
|
|
||||||
|
export class DictationStreamManager {
|
||||||
|
/**
|
||||||
|
* @param {object} params
|
||||||
|
* @param {(msg: { type: string, payload: object }) => void} params.emit
|
||||||
|
* @param {(startOptions: object) => Promise<{ session: object } | { error: string, retryable: boolean, reasonCode?: string }>} params.createSttSession
|
||||||
|
* Resolves a connected streaming transcription session for one dictation.
|
||||||
|
* The streaming transcription session contract:
|
||||||
|
* { requiredSampleRate, appendPcm16(buf), commit(), clear(), close(), on(event, handler) }
|
||||||
|
* @param {number} [params.finalTimeoutMs]
|
||||||
|
* @param {number} [params.autoCommitSeconds]
|
||||||
|
*/
|
||||||
|
constructor({ emit, createSttSession, finalTimeoutMs, autoCommitSeconds }) {
|
||||||
|
this.emit = emit;
|
||||||
|
this.createSttSession = createSttSession;
|
||||||
|
this.finalTimeoutMs = finalTimeoutMs ?? DEFAULT_FINAL_TIMEOUT_MS;
|
||||||
|
this.autoCommitSeconds = autoCommitSeconds ?? DEFAULT_AUTO_COMMIT_SECONDS;
|
||||||
|
this.streams = new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupAll() {
|
||||||
|
for (const dictationId of Array.from(this.streams.keys())) {
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} dictationId
|
||||||
|
* @param {string} format e.g. "audio/pcm;rate=16000;bits=16"
|
||||||
|
* @param {object} startOptions provider/config options forwarded to createSttSession
|
||||||
|
*/
|
||||||
|
async handleStart(dictationId, format, startOptions = {}) {
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
|
||||||
|
const inputRate = parsePcmRateFromFormat(format, 16000) ?? 16000;
|
||||||
|
if (!Number.isFinite(inputRate) || inputRate <= 0) {
|
||||||
|
this.failStream(dictationId, `Invalid dictation input rate in format: ${format}`, false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let resolved;
|
||||||
|
try {
|
||||||
|
resolved = await this.createSttSession(startOptions);
|
||||||
|
} catch (error) {
|
||||||
|
this.failStream(dictationId, error?.message || String(error), true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!resolved || resolved.error) {
|
||||||
|
this.failStream(
|
||||||
|
dictationId,
|
||||||
|
resolved?.error || 'Dictation STT not configured',
|
||||||
|
Boolean(resolved?.retryable),
|
||||||
|
resolved?.reasonCode,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stt = resolved.session;
|
||||||
|
|
||||||
|
stt.on('committed', ({ segmentId }) => {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
state.committedSegmentIds.push(segmentId);
|
||||||
|
state.bytesSinceCommit = 0;
|
||||||
|
state.peakSinceCommit = 0;
|
||||||
|
|
||||||
|
if (state.finishRequested && state.awaitingFinalCommit) {
|
||||||
|
state.awaitingFinalCommit = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.maybeFinalizeStream(dictationId);
|
||||||
|
});
|
||||||
|
|
||||||
|
stt.on('transcript', ({ segmentId, transcript, isFinal }) => {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
state.transcriptsBySegmentId.set(segmentId, transcript);
|
||||||
|
if (isFinal) {
|
||||||
|
state.finalTranscriptSegmentIds.add(segmentId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.finishRequested && state.awaitingFinalCommit && isFinal) {
|
||||||
|
state.awaitingFinalCommit = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const orderedIds = state.committedSegmentIds.includes(segmentId)
|
||||||
|
? state.committedSegmentIds
|
||||||
|
: [...state.committedSegmentIds, segmentId];
|
||||||
|
const partialText = orderedIds
|
||||||
|
.map((id) => state.transcriptsBySegmentId.get(id) ?? '')
|
||||||
|
.join(' ')
|
||||||
|
.trim();
|
||||||
|
this.emit({ type: 'partial', payload: { dictationId, text: partialText } });
|
||||||
|
|
||||||
|
this.maybeSealStreamFinish(dictationId);
|
||||||
|
this.maybeFinalizeStream(dictationId);
|
||||||
|
});
|
||||||
|
|
||||||
|
stt.on('error', (err) => {
|
||||||
|
const message = err?.message || String(err);
|
||||||
|
this.failAndCleanupStream(dictationId, message, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
this.streams.set(dictationId, {
|
||||||
|
dictationId,
|
||||||
|
inputFormat: format,
|
||||||
|
stt,
|
||||||
|
inputRate,
|
||||||
|
outputRate: stt.requiredSampleRate,
|
||||||
|
resampler:
|
||||||
|
inputRate === stt.requiredSampleRate
|
||||||
|
? null
|
||||||
|
: new Pcm16MonoResampler({ inputRate, outputRate: stt.requiredSampleRate }),
|
||||||
|
receivedChunks: new Map(),
|
||||||
|
nextSeqToForward: 0,
|
||||||
|
ackSeq: -1,
|
||||||
|
autoCommitBytes:
|
||||||
|
this.autoCommitSeconds > 0
|
||||||
|
? Math.max(1, Math.round(this.autoCommitSeconds * stt.requiredSampleRate * 2))
|
||||||
|
: 0,
|
||||||
|
bytesSinceCommit: 0,
|
||||||
|
peakSinceCommit: 0,
|
||||||
|
committedSegmentIds: [],
|
||||||
|
transcriptsBySegmentId: new Map(),
|
||||||
|
finalTranscriptSegmentIds: new Set(),
|
||||||
|
awaitingFinalCommit: false,
|
||||||
|
finishRequested: false,
|
||||||
|
finishSealed: false,
|
||||||
|
finalSeq: null,
|
||||||
|
finalTimeout: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.emitAck(dictationId, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {{ dictationId: string, seq: number, audioBase64: string }} params
|
||||||
|
*/
|
||||||
|
handleChunk({ dictationId, seq, audioBase64 }) {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
this.failStream(dictationId, 'Dictation stream not started', true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Number.isInteger(seq) || seq < 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (seq < state.nextSeqToForward) {
|
||||||
|
this.emitAck(dictationId, state.ackSeq);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!state.receivedChunks.has(seq)) {
|
||||||
|
let chunk;
|
||||||
|
try {
|
||||||
|
chunk = Buffer.from(audioBase64, 'base64');
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (chunk.length % 2 !== 0) {
|
||||||
|
chunk = chunk.subarray(0, chunk.length - 1);
|
||||||
|
}
|
||||||
|
state.receivedChunks.set(seq, chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
while (state.receivedChunks.has(state.nextSeqToForward)) {
|
||||||
|
const nextSeq = state.nextSeqToForward;
|
||||||
|
const pcm16 = state.receivedChunks.get(nextSeq);
|
||||||
|
state.receivedChunks.delete(nextSeq);
|
||||||
|
|
||||||
|
const resampled = state.resampler ? state.resampler.processChunk(pcm16) : pcm16;
|
||||||
|
if (resampled.length > 0) {
|
||||||
|
state.stt.appendPcm16(resampled);
|
||||||
|
state.bytesSinceCommit += resampled.length;
|
||||||
|
state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled));
|
||||||
|
try {
|
||||||
|
this.maybeAutoCommitSegment(state);
|
||||||
|
} catch (error) {
|
||||||
|
this.failAndCleanupStream(dictationId, error?.message || String(error), true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
state.nextSeqToForward += 1;
|
||||||
|
state.ackSeq = state.nextSeqToForward - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.emitAck(dictationId, state.ackSeq);
|
||||||
|
this.maybeSealStreamFinish(dictationId);
|
||||||
|
this.maybeFinalizeStream(dictationId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} dictationId
|
||||||
|
* @param {number} finalSeq highest seq the client sent (or -1 if none)
|
||||||
|
*/
|
||||||
|
handleFinish(dictationId, finalSeq) {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
this.failStream(dictationId, 'Dictation stream not started', true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.finishRequested = true;
|
||||||
|
state.finalSeq = finalSeq;
|
||||||
|
|
||||||
|
if (
|
||||||
|
finalSeq >= 0 &&
|
||||||
|
state.ackSeq < 0 &&
|
||||||
|
state.nextSeqToForward === 0 &&
|
||||||
|
state.receivedChunks.size === 0
|
||||||
|
) {
|
||||||
|
this.failStream(
|
||||||
|
dictationId,
|
||||||
|
'Dictation finished but no audio chunks were received',
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.maybeSealStreamFinish(dictationId);
|
||||||
|
this.maybeFinalizeStream(dictationId);
|
||||||
|
|
||||||
|
const updatedState = this.streams.get(dictationId);
|
||||||
|
if (!updatedState) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeoutMs = this.estimateFinalizationTimeout(updatedState);
|
||||||
|
if (updatedState.finalTimeout) {
|
||||||
|
clearTimeout(updatedState.finalTimeout);
|
||||||
|
}
|
||||||
|
updatedState.finalTimeout = setTimeout(() => {
|
||||||
|
this.failAndCleanupStream(dictationId, 'Timed out waiting for final transcription', true);
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
this.emit({ type: 'finish_accepted', payload: { dictationId, timeoutMs } });
|
||||||
|
}
|
||||||
|
|
||||||
|
handleCancel(dictationId) {
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
}
|
||||||
|
|
||||||
|
emitAck(dictationId, ackSeq) {
|
||||||
|
this.emit({ type: 'ack', payload: { dictationId, ackSeq } });
|
||||||
|
}
|
||||||
|
|
||||||
|
failStream(dictationId, error, retryable, reasonCode) {
|
||||||
|
this.emit({
|
||||||
|
type: 'error',
|
||||||
|
payload: {
|
||||||
|
dictationId,
|
||||||
|
error,
|
||||||
|
retryable,
|
||||||
|
...(reasonCode ? { reasonCode } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
failAndCleanupStream(dictationId, error, retryable) {
|
||||||
|
this.failStream(dictationId, error, retryable);
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupStream(dictationId) {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.finalTimeout) {
|
||||||
|
clearTimeout(state.finalTimeout);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
state.stt.close();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
this.streams.delete(dictationId);
|
||||||
|
}
|
||||||
|
|
||||||
|
estimateFinalizationTimeout(state) {
|
||||||
|
const bytesPerSecond = Math.max(1, state.outputRate * 2);
|
||||||
|
const pendingCommittedSegments = state.committedSegmentIds.reduce((count, segmentId) => {
|
||||||
|
return state.finalTranscriptSegmentIds.has(segmentId) ? count : count + 1;
|
||||||
|
}, 0);
|
||||||
|
const committedSet = new Set(state.committedSegmentIds);
|
||||||
|
const pendingUncommittedTranscriptSegments = Array.from(
|
||||||
|
state.transcriptsBySegmentId.keys(),
|
||||||
|
).reduce((count, segmentId) => {
|
||||||
|
if (committedSet.has(segmentId)) {
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
return state.finalTranscriptSegmentIds.has(segmentId) ? count : count + 1;
|
||||||
|
}, 0);
|
||||||
|
const pendingSegments =
|
||||||
|
pendingCommittedSegments +
|
||||||
|
pendingUncommittedTranscriptSegments +
|
||||||
|
(state.awaitingFinalCommit ? 1 : 0);
|
||||||
|
const pendingAudioSeconds = Math.ceil(Math.max(0, state.bytesSinceCommit) / bytesPerSecond);
|
||||||
|
const missingSeqCount =
|
||||||
|
state.finalSeq === null ? 0 : Math.max(0, state.finalSeq - state.ackSeq);
|
||||||
|
|
||||||
|
const extraMs =
|
||||||
|
pendingSegments * FINAL_TIMEOUT_PER_PENDING_SEGMENT_MS +
|
||||||
|
pendingAudioSeconds * FINAL_TIMEOUT_PER_PENDING_AUDIO_SECOND_MS +
|
||||||
|
missingSeqCount * FINAL_TIMEOUT_PER_MISSING_SEQ_MS;
|
||||||
|
|
||||||
|
return Math.max(
|
||||||
|
this.finalTimeoutMs,
|
||||||
|
Math.min(FINAL_TIMEOUT_MAX_MS, this.finalTimeoutMs + extraMs),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
maybeAutoCommitSegment(state) {
|
||||||
|
if (state.finishRequested) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.autoCommitBytes <= 0 || state.bytesSinceCommit < state.autoCommitBytes) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.peakSinceCommit < SILENCE_PEAK_THRESHOLD) {
|
||||||
|
state.stt.clear();
|
||||||
|
state.bytesSinceCommit = 0;
|
||||||
|
state.peakSinceCommit = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.bytesSinceCommit = 0;
|
||||||
|
state.peakSinceCommit = 0;
|
||||||
|
state.stt.commit();
|
||||||
|
}
|
||||||
|
|
||||||
|
maybeSealStreamFinish(dictationId) {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!state.finishRequested || state.finalSeq === null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.ackSeq < state.finalSeq) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.finishSealed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.bytesSinceCommit > 0) {
|
||||||
|
if (state.peakSinceCommit < SILENCE_PEAK_THRESHOLD) {
|
||||||
|
state.stt.clear();
|
||||||
|
state.bytesSinceCommit = 0;
|
||||||
|
state.peakSinceCommit = 0;
|
||||||
|
state.awaitingFinalCommit = false;
|
||||||
|
this.dropUncommittedNonFinalTranscripts(state);
|
||||||
|
} else {
|
||||||
|
state.awaitingFinalCommit = true;
|
||||||
|
try {
|
||||||
|
state.stt.commit();
|
||||||
|
} catch (error) {
|
||||||
|
this.failAndCleanupStream(dictationId, error?.message || String(error), true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
state.awaitingFinalCommit = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.finishSealed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
dropUncommittedNonFinalTranscripts(state) {
|
||||||
|
const committedSet = new Set(state.committedSegmentIds);
|
||||||
|
for (const segmentId of Array.from(state.transcriptsBySegmentId.keys())) {
|
||||||
|
if (committedSet.has(segmentId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (state.finalTranscriptSegmentIds.has(segmentId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
state.transcriptsBySegmentId.delete(segmentId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
maybeFinalizeStream(dictationId) {
|
||||||
|
const state = this.streams.get(dictationId);
|
||||||
|
if (!state) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!state.finishRequested || state.finalSeq === null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.ackSeq < state.finalSeq) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (state.awaitingFinalCommit) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const committedSet = new Set(state.committedSegmentIds);
|
||||||
|
const orderedSegmentIds = [...state.committedSegmentIds];
|
||||||
|
for (const segmentId of state.transcriptsBySegmentId.keys()) {
|
||||||
|
if (!committedSet.has(segmentId)) {
|
||||||
|
orderedSegmentIds.push(segmentId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (orderedSegmentIds.length === 0) {
|
||||||
|
this.emit({ type: 'final', payload: { dictationId, text: '' } });
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const allTranscriptsReady = orderedSegmentIds.every((segmentId) =>
|
||||||
|
state.finalTranscriptSegmentIds.has(segmentId),
|
||||||
|
);
|
||||||
|
if (!allTranscriptsReady) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const orderedText = orderedSegmentIds
|
||||||
|
.map((segmentId) => state.transcriptsBySegmentId.get(segmentId) ?? '')
|
||||||
|
.join(' ')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
this.emit({ type: 'final', payload: { dictationId, text: orderedText } });
|
||||||
|
this.cleanupStream(dictationId);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
import { describe, it, expect } from 'bun:test';
|
||||||
|
import { EventEmitter } from 'events';
|
||||||
|
|
||||||
|
import { DictationStreamManager } from './stream-manager.js';
|
||||||
|
|
||||||
|
const FORMAT = 'audio/pcm;rate=16000;bits=16';
|
||||||
|
|
||||||
|
class FakeSttSession extends EventEmitter {
|
||||||
|
constructor({ transcriptBySegment = () => 'hello world' } = {}) {
|
||||||
|
super();
|
||||||
|
this.requiredSampleRate = 16000;
|
||||||
|
this.appended = [];
|
||||||
|
this.commits = 0;
|
||||||
|
this.clears = 0;
|
||||||
|
this.closed = false;
|
||||||
|
this.segmentCounter = 0;
|
||||||
|
this.transcriptBySegment = transcriptBySegment;
|
||||||
|
}
|
||||||
|
|
||||||
|
async connect() {}
|
||||||
|
|
||||||
|
appendPcm16(buf) {
|
||||||
|
this.appended.push(buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
commit() {
|
||||||
|
this.commits += 1;
|
||||||
|
const segmentId = `seg-${this.segmentCounter}`;
|
||||||
|
this.segmentCounter += 1;
|
||||||
|
this.emit('committed', { segmentId, previousSegmentId: null });
|
||||||
|
setTimeout(() => {
|
||||||
|
this.emit('transcript', {
|
||||||
|
segmentId,
|
||||||
|
transcript: this.transcriptBySegment(segmentId),
|
||||||
|
isFinal: true,
|
||||||
|
});
|
||||||
|
}, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
this.clears += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
close() {
|
||||||
|
this.closed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function loudChunkBase64(samples = 1600, amplitude = 8000) {
|
||||||
|
const arr = new Int16Array(samples);
|
||||||
|
for (let i = 0; i < samples; i += 1) {
|
||||||
|
arr[i] = i % 2 === 0 ? amplitude : -amplitude;
|
||||||
|
}
|
||||||
|
return Buffer.from(arr.buffer).toString('base64');
|
||||||
|
}
|
||||||
|
|
||||||
|
function silentChunkBase64(samples = 1600) {
|
||||||
|
return Buffer.from(new Int16Array(samples).buffer).toString('base64');
|
||||||
|
}
|
||||||
|
|
||||||
|
function createManager(session) {
|
||||||
|
const messages = [];
|
||||||
|
const manager = new DictationStreamManager({
|
||||||
|
emit: (msg) => messages.push(msg),
|
||||||
|
createSttSession: async () => ({ session }),
|
||||||
|
});
|
||||||
|
return { manager, messages };
|
||||||
|
}
|
||||||
|
|
||||||
|
function waitFor(predicate, timeoutMs = 1000) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const tick = () => {
|
||||||
|
if (predicate()) {
|
||||||
|
resolve(undefined);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Date.now() - startedAt > timeoutMs) {
|
||||||
|
reject(new Error('waitFor timed out'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setTimeout(tick, 5);
|
||||||
|
};
|
||||||
|
tick();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('DictationStreamManager', () => {
|
||||||
|
it('transcribes ordered chunks and emits final text', async () => {
|
||||||
|
const session = new FakeSttSession();
|
||||||
|
const { manager, messages } = createManager(session);
|
||||||
|
|
||||||
|
await manager.handleStart('d1', FORMAT, {});
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64() });
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: loudChunkBase64() });
|
||||||
|
manager.handleFinish('d1', 1);
|
||||||
|
|
||||||
|
await waitFor(() => messages.some((m) => m.type === 'final'));
|
||||||
|
|
||||||
|
const final = messages.find((m) => m.type === 'final');
|
||||||
|
expect(final.payload.text).toBe('hello world');
|
||||||
|
expect(session.commits).toBe(1);
|
||||||
|
expect(session.closed).toBe(true);
|
||||||
|
|
||||||
|
const acks = messages.filter((m) => m.type === 'ack');
|
||||||
|
expect(acks[acks.length - 1].payload.ackSeq).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reorders out-of-order chunks before appending', async () => {
|
||||||
|
const session = new FakeSttSession();
|
||||||
|
const { manager, messages } = createManager(session);
|
||||||
|
|
||||||
|
await manager.handleStart('d1', FORMAT, {});
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: loudChunkBase64() });
|
||||||
|
expect(session.appended.length).toBe(0);
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64() });
|
||||||
|
expect(session.appended.length).toBe(2);
|
||||||
|
manager.handleFinish('d1', 1);
|
||||||
|
|
||||||
|
await waitFor(() => messages.some((m) => m.type === 'final'));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears silence-only tails instead of committing', async () => {
|
||||||
|
const session = new FakeSttSession();
|
||||||
|
const { manager, messages } = createManager(session);
|
||||||
|
|
||||||
|
await manager.handleStart('d1', FORMAT, {});
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: silentChunkBase64() });
|
||||||
|
manager.handleFinish('d1', 0);
|
||||||
|
|
||||||
|
await waitFor(() => messages.some((m) => m.type === 'final'));
|
||||||
|
|
||||||
|
const final = messages.find((m) => m.type === 'final');
|
||||||
|
expect(final.payload.text).toBe('');
|
||||||
|
expect(session.commits).toBe(0);
|
||||||
|
expect(session.clears).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fails fast when finish arrives with no chunks', async () => {
|
||||||
|
const session = new FakeSttSession();
|
||||||
|
const { manager, messages } = createManager(session);
|
||||||
|
|
||||||
|
await manager.handleStart('d1', FORMAT, {});
|
||||||
|
manager.handleFinish('d1', 3);
|
||||||
|
|
||||||
|
const error = messages.find((m) => m.type === 'error');
|
||||||
|
expect(error).toBeDefined();
|
||||||
|
expect(error.payload.retryable).toBe(true);
|
||||||
|
expect(session.closed).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reports provider readiness errors from createSttSession', async () => {
|
||||||
|
const messages = [];
|
||||||
|
const manager = new DictationStreamManager({
|
||||||
|
emit: (msg) => messages.push(msg),
|
||||||
|
createSttSession: async () => ({
|
||||||
|
error: 'Dictation model is downloading',
|
||||||
|
retryable: true,
|
||||||
|
reasonCode: 'model_download_in_progress',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
await manager.handleStart('d1', FORMAT, {});
|
||||||
|
const error = messages.find((m) => m.type === 'error');
|
||||||
|
expect(error.payload.reasonCode).toBe('model_download_in_progress');
|
||||||
|
expect(error.payload.retryable).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('emits partials as segment transcripts arrive', async () => {
|
||||||
|
let segment = 0;
|
||||||
|
const session = new FakeSttSession({
|
||||||
|
transcriptBySegment: () => {
|
||||||
|
segment += 1;
|
||||||
|
return segment === 1 ? 'first part' : 'second part';
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { manager, messages } = createManager(session);
|
||||||
|
// Force auto-commit after ~0.05s of audio so two segments form.
|
||||||
|
manager.autoCommitSeconds = 0.05;
|
||||||
|
|
||||||
|
await manager.handleStart('d1', FORMAT, {});
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(1600) });
|
||||||
|
await waitFor(() => session.commits >= 1);
|
||||||
|
manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: loudChunkBase64(1600) });
|
||||||
|
manager.handleFinish('d1', 1);
|
||||||
|
|
||||||
|
await waitFor(() => messages.some((m) => m.type === 'final'));
|
||||||
|
|
||||||
|
const final = messages.find((m) => m.type === 'final');
|
||||||
|
expect(final.payload.text).toBe('first part second part');
|
||||||
|
const partials = messages.filter((m) => m.type === 'partial');
|
||||||
|
expect(partials.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -721,10 +721,18 @@ export const createSettingsHelpers = (dependencies) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (typeof candidate.dictationEnabled === 'boolean') {
|
||||||
|
result.dictationEnabled = candidate.dictationEnabled;
|
||||||
|
}
|
||||||
if (typeof candidate.sttProvider === 'string') {
|
if (typeof candidate.sttProvider === 'string') {
|
||||||
const provider = candidate.sttProvider.trim();
|
const provider = candidate.sttProvider.trim();
|
||||||
if (provider === 'browser' || provider === 'server' || provider === 'wasm') {
|
if (provider === 'local' || provider === 'openai-compatible') {
|
||||||
result.sttProvider = provider;
|
result.sttProvider = provider;
|
||||||
|
} else if (provider === 'server') {
|
||||||
|
// Legacy provider migration: 'server' was the OpenAI-compatible endpoint.
|
||||||
|
result.sttProvider = 'openai-compatible';
|
||||||
|
} else if (provider === 'browser' || provider === 'wasm') {
|
||||||
|
result.sttProvider = 'local';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (typeof candidate.sttServerUrl === 'string') {
|
if (typeof candidate.sttServerUrl === 'string') {
|
||||||
@@ -739,10 +747,10 @@ export const createSettingsHelpers = (dependencies) => {
|
|||||||
result.sttModel = trimmed;
|
result.sttModel = trimmed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (typeof candidate.wasmSttModel === 'string') {
|
if (typeof candidate.sttLocalModel === 'string') {
|
||||||
const trimmed = candidate.wasmSttModel.trim();
|
const trimmed = candidate.sttLocalModel.trim();
|
||||||
if (trimmed.length <= 256) {
|
if (trimmed.length <= STT_MODEL_MAX_LENGTH) {
|
||||||
result.wasmSttModel = trimmed;
|
result.sttLocalModel = trimmed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (typeof candidate.sttLanguage === 'string') {
|
if (typeof candidate.sttLanguage === 'string') {
|
||||||
@@ -751,15 +759,6 @@ export const createSettingsHelpers = (dependencies) => {
|
|||||||
result.sttLanguage = trimmed;
|
result.sttLanguage = trimmed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (typeof candidate.sttSilenceThresholdDb === 'number' && Number.isFinite(candidate.sttSilenceThresholdDb)) {
|
|
||||||
result.sttSilenceThresholdDb = Math.max(-100, Math.min(0, candidate.sttSilenceThresholdDb));
|
|
||||||
}
|
|
||||||
if (typeof candidate.sttSilenceHoldMs === 'number' && Number.isFinite(candidate.sttSilenceHoldMs)) {
|
|
||||||
result.sttSilenceHoldMs = Math.max(250, Math.min(10000, Math.round(candidate.sttSilenceHoldMs)));
|
|
||||||
}
|
|
||||||
if (typeof candidate.sttTranscribeOnStop === 'boolean') {
|
|
||||||
result.sttTranscribeOnStop = candidate.sttTranscribeOnStop;
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
export const createStartupPipelineRuntime = (dependencies) => {
|
export const createStartupPipelineRuntime = (dependencies) => {
|
||||||
const {
|
const {
|
||||||
createTerminalRuntime,
|
createTerminalRuntime,
|
||||||
|
createDictationRuntime,
|
||||||
createMessageStreamWsRuntime,
|
createMessageStreamWsRuntime,
|
||||||
createServerStartupRuntime,
|
createServerStartupRuntime,
|
||||||
} = dependencies;
|
} = dependencies;
|
||||||
@@ -52,6 +53,7 @@ export const createStartupPipelineRuntime = (dependencies) => {
|
|||||||
tunnelRuntimeContext,
|
tunnelRuntimeContext,
|
||||||
attachSignals,
|
attachSignals,
|
||||||
apiOnly,
|
apiOnly,
|
||||||
|
dictationModelsDir,
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
const terminalRuntime = createTerminalRuntime({
|
const terminalRuntime = createTerminalRuntime({
|
||||||
@@ -71,6 +73,16 @@ export const createStartupPipelineRuntime = (dependencies) => {
|
|||||||
TERMINAL_INPUT_WS_MAX_REBINDS_PER_WINDOW: terminalMaxRebindsPerWindow,
|
TERMINAL_INPUT_WS_MAX_REBINDS_PER_WINDOW: terminalMaxRebindsPerWindow,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const dictationRuntime = createDictationRuntime({
|
||||||
|
app,
|
||||||
|
server,
|
||||||
|
express,
|
||||||
|
uiAuthController,
|
||||||
|
isRequestOriginAllowed,
|
||||||
|
rejectWebSocketUpgrade,
|
||||||
|
modelsDir: dictationModelsDir,
|
||||||
|
});
|
||||||
|
|
||||||
const messageStreamRuntime = createMessageStreamWsRuntime({
|
const messageStreamRuntime = createMessageStreamWsRuntime({
|
||||||
server,
|
server,
|
||||||
uiAuthController,
|
uiAuthController,
|
||||||
@@ -125,6 +137,7 @@ export const createStartupPipelineRuntime = (dependencies) => {
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
terminalRuntime,
|
terminalRuntime,
|
||||||
|
dictationRuntime,
|
||||||
messageStreamRuntime,
|
messageStreamRuntime,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,6 +1,16 @@
|
|||||||
export const createRequestSecurityRuntime = (deps) => {
|
export const createRequestSecurityRuntime = (deps) => {
|
||||||
const { readSettingsFromDiskMigrated } = deps;
|
const { readSettingsFromDiskMigrated } = deps;
|
||||||
const packagedClientOrigins = new Set(['openchamber-ui://app', 'capacitor://localhost']);
|
// Origins of packaged (non-browser) clients whose WebView origin never
|
||||||
|
// matches the server host: the desktop shell, the iOS Capacitor WebView
|
||||||
|
// (capacitor://localhost), and the Android Capacitor WebView, which uses
|
||||||
|
// androidScheme 'https' and therefore reports 'https://localhost'. Missing
|
||||||
|
// the Android origin 403'd every WebSocket upgrade from the Android app
|
||||||
|
// (message stream, terminal, dictation) while SSE kept working.
|
||||||
|
const packagedClientOrigins = new Set([
|
||||||
|
'openchamber-ui://app',
|
||||||
|
'capacitor://localhost',
|
||||||
|
'https://localhost',
|
||||||
|
]);
|
||||||
|
|
||||||
const getUiSessionTokenFromRequest = (req) => {
|
const getUiSessionTokenFromRequest = (req) => {
|
||||||
const cookieHeader = req?.headers?.cookie;
|
const cookieHeader = req?.headers?.cookie;
|
||||||
|
|||||||
@@ -24,5 +24,26 @@ describe('request security runtime', () => {
|
|||||||
},
|
},
|
||||||
socket: {},
|
socket: {},
|
||||||
})).resolves.toBe(true);
|
})).resolves.toBe(true);
|
||||||
|
|
||||||
|
// Android Capacitor WebView (androidScheme 'https') reports this origin.
|
||||||
|
await expect(runtime.isRequestOriginAllowed({
|
||||||
|
headers: {
|
||||||
|
origin: 'https://localhost',
|
||||||
|
host: '192.168.1.130:1202',
|
||||||
|
},
|
||||||
|
socket: {},
|
||||||
|
})).resolves.toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rejects unknown origins', async () => {
|
||||||
|
const runtime = createRuntime();
|
||||||
|
|
||||||
|
await expect(runtime.isRequestOriginAllowed({
|
||||||
|
headers: {
|
||||||
|
origin: 'https://evil.example.com',
|
||||||
|
host: '192.168.1.130:1202',
|
||||||
|
},
|
||||||
|
socket: {},
|
||||||
|
})).resolves.toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -310,6 +310,7 @@ const isUrlAuthWebSocketPath = (pathname) => {
|
|||||||
|| pathname === '/api/global/event/ws'
|
|| pathname === '/api/global/event/ws'
|
||||||
|| pathname === '/api/openchamber/realtime-proxy/ws'
|
|| pathname === '/api/openchamber/realtime-proxy/ws'
|
||||||
|| pathname === '/api/terminal/ws'
|
|| pathname === '/api/terminal/ws'
|
||||||
|
|| pathname === '/api/dictation/ws'
|
||||||
|| pathname.startsWith('/api/preview/proxy/');
|
|| pathname.startsWith('/api/preview/proxy/');
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -213,6 +213,28 @@ describe('ui auth client credential seam', () => {
|
|||||||
});
|
});
|
||||||
expect(mountedServeCalled).toBe(true);
|
expect(mountedServeCalled).toBe(true);
|
||||||
|
|
||||||
|
const dictationWsReq = {
|
||||||
|
method: 'GET',
|
||||||
|
path: '/api/dictation/ws',
|
||||||
|
url: `/api/dictation/ws?oc_url_token=${encodeURIComponent(urlToken)}`,
|
||||||
|
headers: { upgrade: 'websocket' },
|
||||||
|
};
|
||||||
|
expect(await auth.ensureSessionToken(dictationWsReq, null)).toBe('client:device-1');
|
||||||
|
|
||||||
|
const dictationHttpReq = {
|
||||||
|
method: 'GET',
|
||||||
|
path: '/api/dictation/ws',
|
||||||
|
url: `/api/dictation/ws?oc_url_token=${encodeURIComponent(urlToken)}`,
|
||||||
|
headers: { accept: 'application/json' },
|
||||||
|
};
|
||||||
|
const dictationHttpRes = createResponse();
|
||||||
|
let dictationHttpCalled = false;
|
||||||
|
await auth.requireAuth(dictationHttpReq, dictationHttpRes, () => {
|
||||||
|
dictationHttpCalled = true;
|
||||||
|
});
|
||||||
|
expect(dictationHttpCalled).toBe(false);
|
||||||
|
expect(dictationHttpRes.statusCode).toBe(401);
|
||||||
|
|
||||||
const arbitraryGetReq = { method: 'GET', path: '/api/config/settings', url: `/api/config/settings?oc_url_token=${encodeURIComponent(urlToken)}`, headers: { accept: 'application/json' } };
|
const arbitraryGetReq = { method: 'GET', path: '/api/config/settings', url: `/api/config/settings?oc_url_token=${encodeURIComponent(urlToken)}`, headers: { accept: 'application/json' } };
|
||||||
const arbitraryGetRes = createResponse();
|
const arbitraryGetRes = createResponse();
|
||||||
let arbitraryGetCalled = false;
|
let arbitraryGetCalled = false;
|
||||||
|
|||||||
Reference in New Issue
Block a user