From de1b85ac566db954472131b5b2bea3792907bd77 Mon Sep 17 00:00:00 2001 From: Bohdan Triapitsyn Date: Sat, 4 Jul 2026 02:48:07 +0300 Subject: [PATCH] feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider) --- .github/workflows/build-macos-arm64-dmg.yml | 2 +- .github/workflows/oc-review.yml | 2 +- .github/workflows/release-desktop-smoke.yml | 4 +- .github/workflows/release.yml | 8 +- .github/workflows/vscode-extension.yml | 2 +- .gitignore | 1 + bun.lock | 15 + .../android/app/src/main/AndroidManifest.xml | 6 + packages/mobile/ios/App/App/Info.plist | 2 + packages/ui/src/App.tsx | 7 +- packages/ui/src/components/chat/ChatInput.tsx | 57 +- .../dictation/ComposerDictation.tsx | 439 ++ .../openchamber/OpenChamberVisualSettings.tsx | 7 +- .../sections/openchamber/VoiceSettings.tsx | 732 ++- .../ui/src/components/views/SettingsView.tsx | 2 +- .../components/voice/BrowserVoiceButton.tsx | 392 -- .../ui/src/components/voice/VoiceProvider.tsx | 30 - .../components/voice/VoiceStatusIndicator.tsx | 139 - packages/ui/src/components/voice/index.ts | 2 - packages/ui/src/hooks/useBrowserVoice.ts | 1007 ---- packages/ui/src/hooks/useDictation.ts | 408 ++ packages/ui/src/hooks/useKeyboardShortcuts.ts | 12 + packages/ui/src/hooks/useLocalTTS.ts | 266 + packages/ui/src/hooks/useMessageTTS.ts | 15 +- packages/ui/src/hooks/useVoiceContext.ts | 57 - packages/ui/src/lib/desktop.ts | 8 +- .../ui/src/lib/dictation/dictation-client.ts | 447 ++ .../lib/dictation/dictation-stream-sender.ts | 240 + .../dictation/use-dictation-audio-source.ts | 301 + .../ui/src/lib/i18n/messages/en.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/en.ts | 12 + .../ui/src/lib/i18n/messages/es.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/es.ts | 12 + .../ui/src/lib/i18n/messages/fr.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/fr.ts | 12 + .../ui/src/lib/i18n/messages/ja.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/ja.ts | 12 + .../ui/src/lib/i18n/messages/ko.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/ko.ts | 12 + .../ui/src/lib/i18n/messages/pl.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/pl.ts | 12 + .../src/lib/i18n/messages/pt-BR.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/pt-BR.ts | 12 + .../ui/src/lib/i18n/messages/uk.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/uk.ts | 12 + .../src/lib/i18n/messages/zh-CN.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/zh-CN.ts | 12 + .../src/lib/i18n/messages/zh-TW.settings.ts | 23 +- packages/ui/src/lib/i18n/messages/zh-TW.ts | 12 + packages/ui/src/lib/persistence.ts | 56 +- packages/ui/src/lib/settings/search.ts | 14 +- packages/ui/src/lib/shortcuts.ts | 7 + .../ui/src/lib/voice/audioStreamService.ts | 397 -- .../ui/src/lib/voice/contextFormatters.ts | 131 - packages/ui/src/lib/voice/index.ts | 17 - packages/ui/src/lib/voice/summarize.ts | 14 +- packages/ui/src/lib/voice/voiceConfig.ts | 32 - packages/ui/src/lib/voice/voiceHooks.ts | 125 - packages/ui/src/lib/voice/voiceSession.ts | 28 - packages/ui/src/lib/voice/wasmSttService.ts | 555 -- packages/ui/src/lib/voice/wasmSttWorker.ts | 92 - packages/ui/src/stores/useConfigStore.ts | 209 +- packages/ui/src/sync/session-ui-store.ts | 1 - packages/ui/src/sync/sync-context.tsx | 5064 ++++++++--------- packages/ui/src/sync/voice-store.ts | 7 - packages/vscode/webview/main.tsx | 17 + packages/web/.gitignore | 1 + packages/web/package.json | 1 + packages/web/server/index.js | 10 + .../web/server/lib/dictation/DOCUMENTATION.md | 63 + packages/web/server/lib/dictation/audio.js | 195 + .../lib/dictation/local/model-catalog.js | 141 + .../lib/dictation/local/model-downloader.js | 163 + .../lib/dictation/local/sherpa-loader.js | 136 + .../lib/dictation/local/sherpa-recognizer.js | 277 + .../server/lib/dictation/local/sherpa-tts.js | 110 + .../lib/dictation/local/worker-client.js | 352 ++ .../lib/dictation/local/worker-process.js | 197 + .../dictation/openai-compatible-session.js | 98 + packages/web/server/lib/dictation/runtime.js | 278 + packages/web/server/lib/dictation/service.js | 302 + .../server/lib/dictation/stream-manager.js | 461 ++ .../lib/dictation/stream-manager.test.js | 194 + .../server/lib/opencode/settings-helpers.js | 27 +- .../lib/opencode/startup-pipeline-runtime.js | 13 + .../server/lib/security/request-security.js | 12 +- .../lib/security/request-security.test.js | 21 + packages/web/server/lib/ui-auth/ui-auth.js | 1 + .../web/server/lib/ui-auth/ui-auth.test.js | 22 + 89 files changed, 8740 insertions(+), 6061 deletions(-) create mode 100644 packages/ui/src/components/dictation/ComposerDictation.tsx delete mode 100644 packages/ui/src/components/voice/BrowserVoiceButton.tsx delete mode 100644 packages/ui/src/components/voice/VoiceProvider.tsx delete mode 100644 packages/ui/src/components/voice/VoiceStatusIndicator.tsx delete mode 100644 packages/ui/src/components/voice/index.ts delete mode 100644 packages/ui/src/hooks/useBrowserVoice.ts create mode 100644 packages/ui/src/hooks/useDictation.ts create mode 100644 packages/ui/src/hooks/useLocalTTS.ts delete mode 100644 packages/ui/src/hooks/useVoiceContext.ts create mode 100644 packages/ui/src/lib/dictation/dictation-client.ts create mode 100644 packages/ui/src/lib/dictation/dictation-stream-sender.ts create mode 100644 packages/ui/src/lib/dictation/use-dictation-audio-source.ts delete mode 100644 packages/ui/src/lib/voice/audioStreamService.ts delete mode 100644 packages/ui/src/lib/voice/contextFormatters.ts delete mode 100644 packages/ui/src/lib/voice/index.ts delete mode 100644 packages/ui/src/lib/voice/voiceConfig.ts delete mode 100644 packages/ui/src/lib/voice/voiceHooks.ts delete mode 100644 packages/ui/src/lib/voice/voiceSession.ts delete mode 100644 packages/ui/src/lib/voice/wasmSttService.ts delete mode 100644 packages/ui/src/lib/voice/wasmSttWorker.ts delete mode 100644 packages/ui/src/sync/voice-store.ts create mode 100644 packages/web/server/lib/dictation/DOCUMENTATION.md create mode 100644 packages/web/server/lib/dictation/audio.js create mode 100644 packages/web/server/lib/dictation/local/model-catalog.js create mode 100644 packages/web/server/lib/dictation/local/model-downloader.js create mode 100644 packages/web/server/lib/dictation/local/sherpa-loader.js create mode 100644 packages/web/server/lib/dictation/local/sherpa-recognizer.js create mode 100644 packages/web/server/lib/dictation/local/sherpa-tts.js create mode 100644 packages/web/server/lib/dictation/local/worker-client.js create mode 100644 packages/web/server/lib/dictation/local/worker-process.js create mode 100644 packages/web/server/lib/dictation/openai-compatible-session.js create mode 100644 packages/web/server/lib/dictation/runtime.js create mode 100644 packages/web/server/lib/dictation/service.js create mode 100644 packages/web/server/lib/dictation/stream-manager.js create mode 100644 packages/web/server/lib/dictation/stream-manager.test.js diff --git a/.github/workflows/build-macos-arm64-dmg.yml b/.github/workflows/build-macos-arm64-dmg.yml index 19479538..a96c1da8 100644 --- a/.github/workflows/build-macos-arm64-dmg.yml +++ b/.github/workflows/build-macos-arm64-dmg.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20" + node-version: "22" - name: Install dependencies run: bun install --frozen-lockfile diff --git a/.github/workflows/oc-review.yml b/.github/workflows/oc-review.yml index 65e49094..655b6665 100644 --- a/.github/workflows/oc-review.yml +++ b/.github/workflows/oc-review.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Install dependencies run: bun install --frozen-lockfile diff --git a/.github/workflows/release-desktop-smoke.yml b/.github/workflows/release-desktop-smoke.yml index 7686eb7c..e81407d8 100644 --- a/.github/workflows/release-desktop-smoke.yml +++ b/.github/workflows/release-desktop-smoke.yml @@ -65,7 +65,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Install dependencies run: bun install --frozen-lockfile @@ -206,7 +206,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Install dependencies run: bun install --frozen-lockfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68390b3f..30148426 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,7 +90,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' registry-url: 'https://registry.npmjs.org' - name: Install dependencies @@ -141,7 +141,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Install dependencies run: bun install --frozen-lockfile @@ -288,7 +288,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Install dependencies run: bun install --frozen-lockfile @@ -365,7 +365,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Download per-arch latest-mac.yml uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 diff --git a/.github/workflows/vscode-extension.yml b/.github/workflows/vscode-extension.yml index 566549c1..ef9ac3a2 100644 --- a/.github/workflows/vscode-extension.yml +++ b/.github/workflows/vscode-extension.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: '20' + node-version: '22' - name: Install dependencies run: bun install --frozen-lockfile diff --git a/.gitignore b/.gitignore index 305aec4c..8a344a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ data/ workspaces/ *.pid .worktrees/ +test-results/ diff --git a/bun.lock b/bun.lock index 37335213..1e4d7467 100644 --- a/bun.lock +++ b/bun.lock @@ -283,6 +283,7 @@ "openai": "^4.79.0", "qrcode-terminal": "^0.12.0", "reflect-metadata": "^0.2.2", + "sherpa-onnx-node": "1.12.28", "simple-git": "^3.28.0", "web-push": "^3.6.7", "ws": "^8.18.3", @@ -2980,6 +2981,20 @@ "shell-quote": ["shell-quote@1.8.3", "", {}, "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw=="], + "sherpa-onnx-darwin-arm64": ["sherpa-onnx-darwin-arm64@1.13.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9x86Cbf+BDFONdtCPM3cnjvtAW0ER8tMaHK5pVfz+SHPt8GeuwRXaiR/BzcByFBUyxCgmceO09/WMZOCi44P/g=="], + + "sherpa-onnx-darwin-x64": ["sherpa-onnx-darwin-x64@1.13.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-TVQ35g7JIpDPB1lUDdcog+JtI0cI45ZzOnvHXm0DtWs/dgxnJXtWMY3uLRtBbLnysV9j5ljffwZ1IX9VDHsCzQ=="], + + "sherpa-onnx-linux-arm64": ["sherpa-onnx-linux-arm64@1.13.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-uDtZkkoP6QQ/3DHOscCpEZ2WpaiHUQsDpbyYaHURrJ7DbsjqGnS6G8l+R589Ro5Bf282QElzBy3okwxXbt3Kxw=="], + + "sherpa-onnx-linux-x64": ["sherpa-onnx-linux-x64@1.13.3", "", { "os": "linux", "cpu": "x64" }, "sha512-OFVK0GYwKwKNsjxbPmfcLQm/dfA0IwAoiIQJ96s+eFYcDqhlapcY06ocdb7SNluGBcM7xgU5jEW2QXBkMIOEvQ=="], + + "sherpa-onnx-node": ["sherpa-onnx-node@1.12.28", "", { "optionalDependencies": { "sherpa-onnx-darwin-arm64": "^1.12.28", "sherpa-onnx-darwin-x64": "^1.12.28", "sherpa-onnx-linux-arm64": "^1.12.28", "sherpa-onnx-linux-x64": "^1.12.28", "sherpa-onnx-win-ia32": "^1.12.28", "sherpa-onnx-win-x64": "^1.12.28" } }, "sha512-EHSB3EG6hKyXaTNh6GU/bwh6i3dncCH6ZCU2mScNzkxRbVStZ7QmNj0Oo4E9XrGGo9jX9pKg9MPHEPyjdK+ApA=="], + + "sherpa-onnx-win-ia32": ["sherpa-onnx-win-ia32@1.13.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-VDZh1M7Ccx/bkP3WwBCFoJzwAwq+b5nR1KRkYRz5p1w5bfhzfa3ACBGr7vpUt5AGUge4qSLe0MSKXyKtSmy1uA=="], + + "sherpa-onnx-win-x64": ["sherpa-onnx-win-x64@1.13.3", "", { "os": "win32", "cpu": "x64" }, "sha512-ZQzcSmFvZK4jzmtWckqxocDUuEjYnBV2MHrDD21HPTeUMfGdE9yfvuSPpesIVfdzKbQzIQY42RAcfZEGWu0FbQ=="], + "shiki": ["shiki@3.23.0", "", { "dependencies": { "@shikijs/core": "3.23.0", "@shikijs/engine-javascript": "3.23.0", "@shikijs/engine-oniguruma": "3.23.0", "@shikijs/langs": "3.23.0", "@shikijs/themes": "3.23.0", "@shikijs/types": "3.23.0", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA=="], "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], diff --git a/packages/mobile/android/app/src/main/AndroidManifest.xml b/packages/mobile/android/app/src/main/AndroidManifest.xml index 0229e443..9ef6ac5c 100644 --- a/packages/mobile/android/app/src/main/AndroidManifest.xml +++ b/packages/mobile/android/app/src/main/AndroidManifest.xml @@ -53,4 +53,10 @@ + + + + diff --git a/packages/mobile/ios/App/App/Info.plist b/packages/mobile/ios/App/App/Info.plist index 3ffde0d6..2d96c874 100644 --- a/packages/mobile/ios/App/App/Info.plist +++ b/packages/mobile/ios/App/App/Info.plist @@ -35,6 +35,8 @@ OpenChamber connects to OpenChamber servers on your local network. NSCameraUsageDescription OpenChamber uses the camera to scan a server's pairing QR code. + NSMicrophoneUsageDescription + OpenChamber uses the microphone for voice dictation in the chat composer. CFBundleURLTypes diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsx index 842e8372..9159a9e0 100644 --- a/packages/ui/src/App.tsx +++ b/packages/ui/src/App.tsx @@ -44,7 +44,6 @@ import { ConfigUpdateOverlay } from '@/components/ui/ConfigUpdateOverlay'; import { AboutDialog } from '@/components/ui/AboutDialog'; import { RuntimeAPIProvider } from '@/contexts/RuntimeAPIProvider'; import { registerRuntimeAPIs } from '@/contexts/runtimeAPIRegistry'; -import { VoiceProvider } from '@/components/voice'; import { useUIStore } from '@/stores/useUIStore'; import { useGitHubAuthStore } from '@/stores/useGitHubAuthStore'; import { useFeatureFlagsStore } from '@/stores/useFeatureFlagsStore'; @@ -928,8 +927,8 @@ function App({ apis }: AppProps) { } // Always mount the full provider tree to avoid remounts when isInitialized - // flips from false → true. FireworksProvider and VoiceProvider are lightweight - // shells; their heavy children are only activated when actually needed. + // flips from false → true. FireworksProvider is a lightweight shell; its + // heavy children are only activated when actually needed. const isBootShell = !isInitialized && !isDesktopRuntime; return ( @@ -937,7 +936,6 @@ function App({ apis }: AppProps) { -
@@ -955,7 +953,6 @@ function App({ apis }: AppProps) { )}
- diff --git a/packages/ui/src/components/chat/ChatInput.tsx b/packages/ui/src/components/chat/ChatInput.tsx index be6bd133..3d32a657 100644 --- a/packages/ui/src/components/chat/ChatInput.tsx +++ b/packages/ui/src/components/chat/ChatInput.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { Textarea } from '@/components/ui/textarea'; -import { BrowserVoiceButton } from '@/components/voice'; +import { ComposerDictation } from '@/components/dictation/ComposerDictation'; // sessionStore removed — currentSessionId comes from useSessionUIStore import { useConfigStore } from '@/stores/useConfigStore'; import { useUIStore } from '@/stores/useUIStore'; @@ -381,7 +381,7 @@ const getProjectIconColor = (projectColor?: string | null): string | undefined = }; const MemoModelControls = React.memo(ModelControls); -const MemoBrowserVoiceButton = React.memo(BrowserVoiceButton); +const MemoComposerDictation = React.memo(ComposerDictation); const MemoMobileAgentButton = React.memo(MobileAgentButton); const MemoMobileModelButton = React.memo(MobileModelButton); const MemoStatusRow = React.memo(StatusRow); @@ -2318,6 +2318,33 @@ const ChatInputComponent: React.FC = ({ onOpenSettings, scrollTo void handleSubmitRef.current(); }, []); + // Dictation: insert the transcript inline; optionally submit immediately. + // getCurrentInputSnapshot reads textareaRef.current.value first, so setting + // it synchronously lets handleSubmit pick up the text in the same tick. + const handleDictationInsert = React.useCallback((text: string) => { + setMessage((prev) => { + const next = appendInlineText(prev, text); + const textarea = textareaRef.current; + if (textarea) { + textarea.value = next; + } + return next; + }); + setTimeout(() => { + textareaRef.current?.focus(); + }, 0); + }, []); + + const handleDictationInsertAndSend = React.useCallback((text: string) => { + const textarea = textareaRef.current; + const next = appendInlineText(textarea?.value ?? messageRef.current, text); + if (textarea) { + textarea.value = next; + } + setMessage(next); + void handleSubmitRef.current(); + }, []); + // Preset chips rendered outside this component (e.g. under the welcome // message on narrow surfaces) request a submit via the input store; consume // it here so it routes through the same command-aware submit path. @@ -4420,6 +4447,9 @@ const ChatInputComponent: React.FC = ({ onOpenSettings, scrollTo : undefined} /> )} + {/* Positioning context for the dictation overlay: covers the + text area + footer exactly, excluding MobileSessionStatusBar. */} +
@@ -4559,7 +4589,16 @@ const ChatInputComponent: React.FC = ({ onOpenSettings, scrollTo />
- + = ({ onOpenSettings, scrollTo
- + = ({ onOpenSettings, scrollTo )}
+
{/* Mobile session panel: slide-up overlay toggled by MobileSessionPanelTrigger. */} {isMobile && } diff --git a/packages/ui/src/components/dictation/ComposerDictation.tsx b/packages/ui/src/components/dictation/ComposerDictation.tsx new file mode 100644 index 00000000..7f3fd913 --- /dev/null +++ b/packages/ui/src/components/dictation/ComposerDictation.tsx @@ -0,0 +1,439 @@ +/** + * Composer dictation controls: a mic button for the composer footer plus a + * full-composer overlay while dictation is active (recording, transcribing, + * or failed). The overlay mirrors the composer's own layout — the transcript + * area uses the same paddings/typography as the textarea and the action row + * reuses the footer icon-button styling — so toggling dictation causes no + * vertical shift. + */ + +import React from 'react'; + +import { Icon } from '@/components/icon/Icon'; +import { useI18n } from '@/lib/i18n'; +import { useThemeSystem } from '@/contexts/useThemeSystem'; +import { cn } from '@/lib/utils'; +import { runtimeFetch } from '@/lib/runtime-fetch'; +import { useDictation } from '@/hooks/useDictation'; +import { isDictationCaptureSupported } from '@/lib/dictation/use-dictation-audio-source'; +import { isVSCodeRuntime } from '@/lib/desktop'; +import { useConfigStore } from '@/stores/useConfigStore'; +import { useUIStore } from '@/stores/useUIStore'; +import { formatShortcutForDisplay, getEffectiveShortcutCombo } from '@/lib/shortcuts'; + +interface ComposerDictationProps { + radius?: number | string; + isMobile: boolean; + footerIconButtonClass: string; + footerPaddingClass: string; + iconSizeClass: string; + sendIconSizeClass: string; + disabled?: boolean; + onInsert: (text: string) => void; + onInsertAndSend: (text: string) => void; +} + +const formatDuration = (seconds: number): string => { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins}:${String(secs).padStart(2, '0')}`; +}; + +const VolumeMeter: React.FC<{ volume: number }> = ({ volume }) => { + const { currentTheme } = useThemeSystem(); + return ( +