From ef85c63336b2f76cb6281e0915e9816dfa5e8e79 Mon Sep 17 00:00:00 2001 From: Bohdan Triapitsyn Date: Thu, 14 May 2026 01:19:52 +0300 Subject: [PATCH] fix: voice input in Electron - local Whisper STT + network error handling - Add local Whisper STT via Transformers.js with Web Worker (no UI freeze) - Default sttProvider to 'local' in Electron (browser STT unavailable) - Fix infinite toast loop: stop auto-restart on network errors - Add retry limit with exponential backoff for transient STT errors - Append voice transcript to input field (append-inline), not replace - Add model catalog with download/load button in Voice Settings --- bun.lock | 85 ++- package.json | 7 +- .../sections/openchamber/VoiceSettings.tsx | 125 +++- .../components/voice/BrowserVoiceButton.tsx | 4 +- packages/ui/src/hooks/useBrowserVoice.ts | 220 ++++--- packages/ui/src/lib/desktop.ts | 3 +- .../ui/src/lib/i18n/messages/en.settings.ts | 8 + .../ui/src/lib/i18n/messages/es.settings.ts | 8 + .../ui/src/lib/i18n/messages/ko.settings.ts | 8 + .../ui/src/lib/i18n/messages/pl.settings.ts | 8 + .../src/lib/i18n/messages/pt-BR.settings.ts | 8 + .../ui/src/lib/i18n/messages/uk.settings.ts | 8 + .../src/lib/i18n/messages/zh-CN.settings.ts | 8 + .../ui/src/lib/voice/browserVoiceService.ts | 11 +- packages/ui/src/lib/voice/wasmSttService.ts | 556 ++++++++++++++++++ packages/ui/src/lib/voice/wasmSttWorker.ts | 92 +++ packages/ui/src/stores/useConfigStore.ts | 36 +- .../server/lib/opencode/settings-helpers.js | 8 +- 18 files changed, 1114 insertions(+), 89 deletions(-) create mode 100644 packages/ui/src/lib/voice/wasmSttService.ts create mode 100644 packages/ui/src/lib/voice/wasmSttWorker.ts diff --git a/bun.lock b/bun.lock index 25c0b200..e82750b2 100644 --- a/bun.lock +++ b/bun.lock @@ -44,6 +44,7 @@ "@radix-ui/react-toggle": "^1.1.10", "@radix-ui/react-tooltip": "^1.2.8", "@types/react-syntax-highlighter": "^15.5.13", + "@xenova/transformers": "^2.17.2", "bun-pty": "^0.4.5", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -763,6 +764,8 @@ "@hexagon/base64": ["@hexagon/base64@1.1.28", "", {}, "sha512-lhqDEAvWixy3bZ+UOYbPwUbBkwBq5C1LAJ/xPC8Oi+lL54oyakv/npbA0aU2hgCsx/1NUd4IBvV03+aUBWxerw=="], + "@huggingface/jinja": ["@huggingface/jinja@0.2.2", "", {}, "sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA=="], + "@humanfs/core": ["@humanfs/core@0.19.1", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="], "@humanfs/node": ["@humanfs/node@0.16.7", "", { "dependencies": { "@humanfs/core": "^0.19.1", "@humanwhocodes/retry": "^0.4.0" } }, "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ=="], @@ -923,6 +926,26 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.1", "", {}, "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="], + "@radix-ui/number": ["@radix-ui/number@1.1.1", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="], "@radix-ui/primitive": ["@radix-ui/primitive@1.1.3", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="], @@ -1253,6 +1276,8 @@ "@types/keyv": ["@types/keyv@3.1.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg=="], + "@types/long": ["@types/long@4.0.2", "", {}, "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA=="], + "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], "@types/methods": ["@types/methods@1.1.4", "", {}, "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ=="], @@ -1359,6 +1384,8 @@ "@vscode/vsce-sign-win32-x64": ["@vscode/vsce-sign-win32-x64@2.0.6", "", { "os": "win32", "cpu": "x64" }, "sha512-mgth9Kvze+u8CruYMmhHw6Zgy3GRX2S+Ed5oSokDEK5vPEwGGKnmuXua9tmFhomeAnhgJnL4DCna3TiNuGrBTQ=="], + "@xenova/transformers": ["@xenova/transformers@2.17.2", "", { "dependencies": { "@huggingface/jinja": "^0.2.2", "onnxruntime-web": "1.14.0", "sharp": "^0.32.0" }, "optionalDependencies": { "onnxruntime-node": "1.14.0" } }, "sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ=="], + "@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="], "@yarnpkg/lockfile": ["@yarnpkg/lockfile@1.1.0", "", {}, "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ=="], @@ -1433,6 +1460,8 @@ "azure-devops-node-api": ["azure-devops-node-api@12.5.0", "", { "dependencies": { "tunnel": "0.0.6", "typed-rest-client": "^1.8.4" } }, "sha512-R5eFskGvOm3U/GzeAuxRkUsAl0hrAwGgWn6zAd2KrZmrEhWZVqLew4OOupbQlXUuojUzpGtq62SmdhJ06N88og=="], + "b4a": ["b4a@1.8.1", "", { "peerDependencies": { "react-native-b4a": "*" }, "optionalPeers": ["react-native-b4a"] }, "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw=="], + "babel-plugin-polyfill-corejs2": ["babel-plugin-polyfill-corejs2@0.4.15", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-define-polyfill-provider": "^0.6.6", "semver": "^6.3.1" }, "peerDependencies": { "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" } }, "sha512-hR3GwrRwHUfYwGfrisXPIDP3JcYfBrW7wKE7+Au6wDYl7fm/ka1NEII6kORzxNU556JjfidZeBsO10kYvtV1aw=="], "babel-plugin-polyfill-corejs3": ["babel-plugin-polyfill-corejs3@0.14.0", "", { "dependencies": { "@babel/helper-define-polyfill-provider": "^0.6.6", "core-js-compat": "^3.48.0" }, "peerDependencies": { "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" } }, "sha512-AvDcMxJ34W4Wgy4KBIIePQTAOP1Ie2WFwkQp3dB7FQ/f0lI5+nM96zUnYEOE1P9sEg0es5VCP0HxiWu5fUHZAQ=="], @@ -1445,6 +1474,18 @@ "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "bare-events": ["bare-events@2.8.2", "", { "peerDependencies": { "bare-abort-controller": "*" }, "optionalPeers": ["bare-abort-controller"] }, "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ=="], + + "bare-fs": ["bare-fs@4.7.1", "", { "dependencies": { "bare-events": "^2.5.4", "bare-path": "^3.0.0", "bare-stream": "^2.6.4", "bare-url": "^2.2.2", "fast-fifo": "^1.3.2" }, "peerDependencies": { "bare-buffer": "*" }, "optionalPeers": ["bare-buffer"] }, "sha512-WDRsyVN52eAx/lBamKD6uyw8H4228h/x0sGGGegOamM2cd7Pag88GfMQalobXI+HaEUxpCkbKQUDOQqt9wawRw=="], + + "bare-os": ["bare-os@3.9.1", "", {}, "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ=="], + + "bare-path": ["bare-path@3.0.0", "", { "dependencies": { "bare-os": "^3.0.1" } }, "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw=="], + + "bare-stream": ["bare-stream@2.13.1", "", { "dependencies": { "streamx": "^2.25.0", "teex": "^1.0.1" }, "peerDependencies": { "bare-abort-controller": "*", "bare-buffer": "*", "bare-events": "*" }, "optionalPeers": ["bare-abort-controller", "bare-buffer", "bare-events"] }, "sha512-Vp0cnjYyrEC4whYTymQ+YZi6pBpfiICZO3cfRG8sy67ZNWe951urv1x4eW1BKNngw3U+3fPYb5JQvHbCtxH7Ow=="], + + "bare-url": ["bare-url@2.4.3", "", { "dependencies": { "bare-path": "^3.0.0" } }, "sha512-Kccpc7ACfXaxfeInfqKcZtW4pT5YBn1mesc4sCsun6sRwtbJ4h+sNOaksUpYEJUKfN65YWC6Bw2OJEFiKxq8nQ=="], + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], "baseline-browser-mapping": ["baseline-browser-mapping@2.10.0", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-lIyg0szRfYbiy67j9KN8IyeD7q7hcmqnJ1ddWmNt19ItGpNN64mnllmxUNFIOdOm6by97jlL6wfpTTJrmnjWAA=="], @@ -1823,6 +1864,8 @@ "eventemitter3": ["eventemitter3@4.0.7", "", {}, "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="], + "events-universal": ["events-universal@1.0.1", "", { "dependencies": { "bare-events": "^2.7.0" } }, "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw=="], + "expand-template": ["expand-template@2.0.3", "", {}, "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="], "expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="], @@ -1845,6 +1888,8 @@ "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + "fast-fifo": ["fast-fifo@1.3.2", "", {}, "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fast-json-stable-stringify": ["fast-json-stable-stringify@2.1.0", "", {}, "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="], @@ -1879,6 +1924,8 @@ "flat-cache": ["flat-cache@4.0.1", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="], + "flatbuffers": ["flatbuffers@1.12.0", "", {}, "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ=="], + "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], @@ -1967,6 +2014,8 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + "guid-typescript": ["guid-typescript@1.0.9", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="], + "has-bigints": ["has-bigints@1.1.0", "", {}, "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg=="], "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], @@ -2291,6 +2340,8 @@ "log-symbols": ["log-symbols@4.1.0", "", { "dependencies": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" } }, "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg=="], + "long": ["long@4.0.0", "", {}, "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="], + "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], "lowercase-keys": ["lowercase-keys@2.0.0", "", {}, "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="], @@ -2527,6 +2578,14 @@ "oniguruma-to-es": ["oniguruma-to-es@4.3.4", "", { "dependencies": { "oniguruma-parser": "^0.12.1", "regex": "^6.0.1", "regex-recursion": "^6.0.2" } }, "sha512-3VhUGN3w2eYxnTzHn+ikMI+fp/96KoRSVK9/kMTcFqj1NRDh2IhQCKvYxDnWePKRXY/AqH+Fuiyb7VHSzBjHfA=="], + "onnx-proto": ["onnx-proto@4.0.4", "", { "dependencies": { "protobufjs": "^6.8.8" } }, "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA=="], + + "onnxruntime-common": ["onnxruntime-common@1.14.0", "", {}, "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew=="], + + "onnxruntime-node": ["onnxruntime-node@1.14.0", "", { "dependencies": { "onnxruntime-common": "~1.14.0" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w=="], + + "onnxruntime-web": ["onnxruntime-web@1.14.0", "", { "dependencies": { "flatbuffers": "^1.12.0", "guid-typescript": "^1.0.9", "long": "^4.0.0", "onnx-proto": "^4.0.4", "onnxruntime-common": "~1.14.0", "platform": "^1.3.6" } }, "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw=="], + "open": ["open@7.4.2", "", { "dependencies": { "is-docker": "^2.0.0", "is-wsl": "^2.1.1" } }, "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q=="], "openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="], @@ -2593,6 +2652,8 @@ "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "plist": ["plist@3.1.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.8", "base64-js": "^1.5.1", "xmlbuilder": "^15.1.1" } }, "sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ=="], "pluralize": ["pluralize@8.0.0", "", {}, "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA=="], @@ -2627,6 +2688,8 @@ "property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + "protobufjs": ["protobufjs@6.11.6", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/long": "^4.0.1", "@types/node": ">=13.7.0", "long": "^4.0.0" }, "bin": { "pbjs": "bin/pbjs", "pbts": "bin/pbts" } }, "sha512-k8BHqgPBOtrlougZZqF2uUk5Z7bN8f0wj+3e8M3hvtSv0NBAz4VBy5f6R5Nxq/l+i7mRFTgNZb2trxqTpHNY/A=="], + "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], "pstree.remy": ["pstree.remy@1.1.8", "", {}, "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w=="], @@ -2809,6 +2872,8 @@ "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], + "sharp": ["sharp@0.32.6", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.2", "node-addon-api": "^6.1.0", "prebuild-install": "^7.1.1", "semver": "^7.5.4", "simple-get": "^4.0.1", "tar-fs": "^3.0.4", "tunnel-agent": "^0.6.0" } }, "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], @@ -2891,6 +2956,8 @@ "stop-iteration-iterator": ["stop-iteration-iterator@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "internal-slot": "^1.1.0" } }, "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ=="], + "streamx": ["streamx@2.25.0", "", { "dependencies": { "events-universal": "^1.0.0", "fast-fifo": "^1.3.2", "text-decoder": "^1.1.0" } }, "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg=="], + "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "string-width-cjs": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], @@ -2949,9 +3016,11 @@ "tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="], - "tar-fs": ["tar-fs@2.1.4", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ=="], + "tar-fs": ["tar-fs@3.1.2", "", { "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" }, "optionalDependencies": { "bare-fs": "^4.0.1", "bare-path": "^3.0.0" } }, "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw=="], - "tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + "tar-stream": ["tar-stream@3.2.0", "", { "dependencies": { "b4a": "^1.6.4", "bare-fs": "^4.5.5", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg=="], + + "teex": ["teex@1.0.1", "", { "dependencies": { "streamx": "^2.12.5" } }, "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg=="], "temp": ["temp@0.9.4", "", { "dependencies": { "mkdirp": "^0.5.1", "rimraf": "~2.6.2" } }, "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA=="], @@ -2965,6 +3034,8 @@ "terser": ["terser@5.46.0", "", { "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, "bin": { "terser": "bin/terser" } }, "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg=="], + "text-decoder": ["text-decoder@1.2.7", "", { "dependencies": { "b4a": "^1.6.4" } }, "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ=="], + "text-table": ["text-table@0.2.0", "", {}, "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw=="], "textextensions": ["textextensions@6.11.0", "", { "dependencies": { "editions": "^6.21.0" } }, "sha512-tXJwSr9355kFJI3lbCkPpUH5cP8/M0GGy2xLO34aZCjMXBaK3SoPnZwr/oWmo1FdCnELcs4npdCIOFtq9W3ruQ=="], @@ -3503,6 +3574,8 @@ "postject/commander": ["commander@9.5.0", "", {}, "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ=="], + "prebuild-install/tar-fs": ["tar-fs@2.1.4", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ=="], + "qrcode/yargs": ["yargs@15.4.1", "", { "dependencies": { "cliui": "^6.0.0", "decamelize": "^1.2.0", "find-up": "^4.1.0", "get-caller-file": "^2.0.1", "require-directory": "^2.1.1", "require-main-filename": "^2.0.0", "set-blocking": "^2.0.0", "string-width": "^4.2.0", "which-module": "^2.0.0", "y18n": "^4.0.0", "yargs-parser": "^18.1.2" } }, "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A=="], "rc/strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="], @@ -3521,6 +3594,8 @@ "serialize-error/type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], + "sharp/node-addon-api": ["node-addon-api@6.1.0", "", {}, "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="], + "slice-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], "slice-ansi/is-fullwidth-code-point": ["is-fullwidth-code-point@4.0.0", "", {}, "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ=="], @@ -3545,8 +3620,6 @@ "table/slice-ansi": ["slice-ansi@4.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", "is-fullwidth-code-point": "^3.0.0" } }, "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ=="], - "tar-fs/chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], - "temp/mkdirp": ["mkdirp@0.5.6", "", { "dependencies": { "minimist": "^1.2.6" }, "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw=="], "temp/rimraf": ["rimraf@2.6.3", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA=="], @@ -3689,6 +3762,10 @@ "openai/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], + "prebuild-install/tar-fs/chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], + + "prebuild-install/tar-fs/tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + "qrcode/yargs/cliui": ["cliui@6.0.0", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", "wrap-ansi": "^6.2.0" } }, "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ=="], "qrcode/yargs/find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="], diff --git a/package.json b/package.json index 2ef37f5b..4625cf67 100644 --- a/package.json +++ b/package.json @@ -68,17 +68,18 @@ "release:test:arm": "./scripts/test-release-build.sh aarch64" }, "dependencies": { + "@base-ui/react": "^1.4.0", "@codemirror/autocomplete": "^6.20.0", "@codemirror/commands": "^6.10.1", + "@codemirror/lang-cpp": "^6.0.3", "@codemirror/lang-css": "^6.3.1", + "@codemirror/lang-go": "^6.0.1", "@codemirror/lang-html": "^6.4.11", "@codemirror/lang-javascript": "^6.2.4", "@codemirror/lang-json": "^6.0.2", "@codemirror/lang-markdown": "^6.5.0", "@codemirror/lang-python": "^6.2.1", "@codemirror/lang-rust": "^6.0.2", - "@codemirror/lang-cpp": "^6.0.3", - "@codemirror/lang-go": "^6.0.1", "@codemirror/lang-sql": "^6.10.0", "@codemirror/lang-xml": "^6.1.0", "@codemirror/lang-yaml": "^6.1.2", @@ -96,7 +97,6 @@ "@lezer/highlight": "^1.2.3", "@octokit/rest": "^22.0.1", "@opencode-ai/sdk": "^1.4.48", - "@base-ui/react": "^1.4.0", "@radix-ui/react-collapsible": "^1.1.12", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", @@ -107,6 +107,7 @@ "@radix-ui/react-toggle": "^1.1.10", "@radix-ui/react-tooltip": "^1.2.8", "@types/react-syntax-highlighter": "^15.5.13", + "@xenova/transformers": "^2.17.2", "bun-pty": "^0.4.5", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx b/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx index fb8619af..ad3bc212 100644 --- a/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx +++ b/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx @@ -17,6 +17,8 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip import { Icon } from "@/components/icon/Icon"; import { browserVoiceService } from '@/lib/voice/browserVoiceService'; import { audioStreamService } from '@/lib/voice/audioStreamService'; +import { wasmSttService, WASM_MODELS } from '@/lib/voice/wasmSttService'; +import type { WasmModelStatus } from '@/lib/voice/wasmSttService'; import { cn } from '@/lib/utils'; import { useI18n } from '@/lib/i18n'; const LANGUAGE_OPTIONS = [ @@ -32,6 +34,82 @@ const LANGUAGE_OPTIONS = [ { value: 'uk-UA', label: 'Українська' }, ]; +const WasmModelStatusIndicator = ({ modelId }: { modelId: string }) => { + const { t } = useI18n(); + const [status, setStatus] = useState(wasmSttService.getModelStatus()); + const [loading, setLoading] = useState(false); + + useEffect(() => { + const handler = (s: WasmModelStatus) => setStatus(s); + wasmSttService.onModelStatusChange = handler; + return () => { wasmSttService.onModelStatusChange = null; }; + }, []); + + const currentModelId = wasmSttService.getCurrentModelId(); + const isLoadingOrDownloading = status.state === 'downloading' || status.state === 'loading'; + + // Reset local loading state when model finishes loading or errors + useEffect(() => { + if (status.state !== 'downloading' && status.state !== 'loading') { + setLoading(false); + } + }, [status.state]); + + const handleDownload = async () => { + setLoading(true); + try { + await wasmSttService.loadModel(modelId); + } catch { + // Error is shown via status indicator + } + }; + + if (status.state === 'ready' && currentModelId === modelId) { + return ( +
+ + {t('settings.voice.page.stt.wasmLoaded')} + +
+ ); + } + + if (isLoadingOrDownloading) { + const progress = status.state === 'downloading' ? Math.round(status.progress) : undefined; + return ( +
+ + {status.state === 'downloading' ? t('settings.voice.page.stt.wasmDownloading') : t('settings.voice.page.stt.wasmLoading')} + {progress !== undefined ? ` (${progress}%)` : ''} + +
+ ); + } + + if (status.state === 'error') { + // Partial download (cached progress): show retry button. + return ( +
+ {status.error} + +
+ ); + } + + return ( +
+ + {t('settings.voice.page.stt.wasmNotLoaded')} + + +
+ ); +}; + const OPENAI_VOICE_OPTIONS = [ { value: 'alloy', label: 'Alloy' }, { value: 'ash', label: 'Ash' }, @@ -86,6 +164,8 @@ export const VoiceSettings: React.FC = () => { const setSttServerUrl = useConfigStore((state) => state.setSttServerUrl); const sttModel = useConfigStore((state) => state.sttModel); const setSttModel = useConfigStore((state) => state.setSttModel); + const wasmSttModel = useConfigStore((state) => state.wasmSttModel); + const setWasmSttModel = useConfigStore((state) => state.setWasmSttModel); const sttLanguage = useConfigStore((state) => state.sttLanguage); const setSttLanguage = useConfigStore((state) => state.setSttLanguage); const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb); @@ -647,12 +727,12 @@ export const VoiceSettings: React.FC = () => { {voiceProvider === 'browser' && filteredBrowserVoices.length > 0 && ( <> - setBrowserVoice(value === '$auto' ? '' : value)}> - {t('settings.voice.page.field.auto')} + {t('settings.voice.page.field.auto')} {filteredBrowserVoices.map((v) => ( {v.name} ({v.lang}) ))} @@ -765,6 +845,15 @@ export const VoiceSettings: React.FC = () => { > {t('settings.voice.page.provider.server')} + @@ -860,6 +949,38 @@ export const VoiceSettings: React.FC = () => { )} + + {sttProvider === 'wasm' && ( +
+
+ + {t('settings.voice.page.stt.wasmModel')} + + +

+ {WASM_MODELS.find((m) => m.id === wasmSttModel)?.description} +

+
+ {/* Model status indicator */} + +
+ )} )} diff --git a/packages/ui/src/components/voice/BrowserVoiceButton.tsx b/packages/ui/src/components/voice/BrowserVoiceButton.tsx index d72abb4b..612dd649 100644 --- a/packages/ui/src/components/voice/BrowserVoiceButton.tsx +++ b/packages/ui/src/components/voice/BrowserVoiceButton.tsx @@ -113,7 +113,9 @@ export function BrowserVoiceButton() { const isIdle = status === 'idle'; const isSpeaking = status === 'speaking'; - const canTranscribeOnStop = sttProvider === 'server' && sttTranscribeOnStop; + // WASM STT always needs finishVoiceInput to flush the recorder and transcribe. + // Server STT uses it when sttTranscribeOnStop is enabled. + const canTranscribeOnStop = sttProvider === 'wasm' || (sttProvider === 'server' && sttTranscribeOnStop); const isListeningWithTranscribeOnStop = status === 'listening' && canTranscribeOnStop; // Show toast notification when voice error occurs diff --git a/packages/ui/src/hooks/useBrowserVoice.ts b/packages/ui/src/hooks/useBrowserVoice.ts index 783c9c2f..20877dae 100644 --- a/packages/ui/src/hooks/useBrowserVoice.ts +++ b/packages/ui/src/hooks/useBrowserVoice.ts @@ -28,6 +28,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { browserVoiceService } from '@/lib/voice/browserVoiceService'; import { audioStreamService } from '@/lib/voice/audioStreamService'; +import { wasmSttService } from '@/lib/voice/wasmSttService'; import { useSessionUIStore } from '@/sync/session-ui-store'; import { useInputStore } from '@/sync/input-store'; import { getSyncMessages, getSyncParts } from '@/sync/sync-refs'; @@ -127,6 +128,9 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { const pendingFinalTranscriptRef = useRef(''); const finalTranscriptTimerRef = useRef | null>(null); const deviceChangeRestartTimerRef = useRef | null>(null); + const consecutiveRecoveryRetriesRef = useRef(0); + const recoveryTimerRef = useRef | null>(null); + const isFinalizingRef = useRef(false); // Store access const currentSessionId = useSessionUIStore((s) => s.currentSessionId); @@ -157,13 +161,16 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { const sttProvider = useConfigStore((state) => state.sttProvider); const sttServerUrl = useConfigStore((state) => state.sttServerUrl); const sttModel = useConfigStore((state) => state.sttModel); + const wasmSttModel = useConfigStore((state) => state.wasmSttModel); const sttLanguage = useConfigStore((state) => state.sttLanguage); const sttSilenceThresholdDb = useConfigStore((state) => state.sttSilenceThresholdDb); const sttSilenceHoldMs = useConfigStore((state) => state.sttSilenceHoldMs); const isSupported = sttProvider === 'server' ? audioStreamService.isSupported() - : browserVoiceService.isSupported(); + : sttProvider === 'wasm' + ? wasmSttService.isSupported() + : browserVoiceService.isSupported(); // Server TTS for mobile (bypasses Safari audio restrictions) const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS({ @@ -287,6 +294,20 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { const handleSpeechErrorRef = useRef<((errorMsg: string) => void) | null>(null); const handleSpeechResultRef = useRef<((text: string, isFinal: boolean) => Promise) | null>(null); + // Start STT via the currently-selected provider. + // Called by auto-recovery, restart-after-TTS, and visibility-resume paths. + const startCurrentSTT = useCallback((lang: string) => { + if (sttProvider === 'server') { + void audioStreamService.startListening(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); + } else if (sttProvider === 'wasm') { + void wasmSttService.startListening(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); + } else if (isMobile) { + browserVoiceService.startListeningSync(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); + } else { + browserVoiceService.startListening(lang, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); + } + }, [sttProvider, isMobile]); + // Handle speech recognition error const handleSpeechError = useCallback((errorMsg: string) => { // Ignore errors if we've already stopped voice mode @@ -316,26 +337,59 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { setStatus('idle'); return; } + + // Network / server-unreachable errors: don't retry at all. + // The user must fix connectivity and then manually restart voice. + const isNetworkError = + normalizedError.includes('network') || + normalizedError.includes('connection') || + normalizedError.includes('check connection'); + + if (isNetworkError) { + console.error('[useBrowserVoice] Network error — staying in error state:', errorMsg); + setError(errorMsg); + setStatus('error'); + consecutiveRecoveryRetriesRef.current = 0; + if (recoveryTimerRef.current !== null) { + clearTimeout(recoveryTimerRef.current); + recoveryTimerRef.current = null; + } + return; + } console.error('[useBrowserVoice] Recognition error:', errorMsg); setError(errorMsg); setStatus('error'); - // Auto-recover from certain errors - if (!errorMsg.includes('permission') && !errorMsg.includes('not allowed')) { - setTimeout(() => { - if (isActiveRef.current) { - setStatus('listening'); - setError(null); - if (sttProvider === 'server') { - audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechError).catch(() => {}); - } else { - browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError); - } - } - }, 1000); + // Auto-recover from transient / non-permission errors with limited retries. + // Skip recovery when finalising — the user explicitly stopped voice. + if (isPermissionStyleError || isFinalizingRef.current) { + consecutiveRecoveryRetriesRef.current = 0; + return; } - }, [language, conversationMode, sttProvider]); + + const nextRetry = consecutiveRecoveryRetriesRef.current + 1; + consecutiveRecoveryRetriesRef.current = nextRetry; + const MAX_RECOVERY_RETRIES = 3; + + if (nextRetry <= MAX_RECOVERY_RETRIES) { + const delay = Math.min(1000 * Math.pow(2, nextRetry - 1), 8000); + console.log(`[useBrowserVoice] Scheduling recovery retry ${nextRetry}/${MAX_RECOVERY_RETRIES} in ${delay}ms`); + + if (recoveryTimerRef.current !== null) { + clearTimeout(recoveryTimerRef.current); + } + recoveryTimerRef.current = setTimeout(() => { + recoveryTimerRef.current = null; + if (!isActiveRef.current) return; + setStatus('listening'); + setError(null); + startCurrentSTT(language); + }, delay); + } else { + console.log('[useBrowserVoice] Max recovery retries reached — staying in error state'); + } + }, [language, conversationMode, startCurrentSTT]); // Update the ref when handleSpeechError changes useEffect(() => { @@ -362,7 +416,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { // Non-continuous mode: fill chat input only, do not auto-send. if (!conversationMode) { - setPendingInputText(finalText.trim(), 'replace'); + setPendingInputText(finalText.trim(), 'append-inline'); processingMessageRef.current = false; isActiveRef.current = false; setStatus('idle'); @@ -446,19 +500,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { } setStatus('listening'); - if (sttProvider === 'server') { - audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => { - console.error('[useBrowserVoice] Failed to restart server STT:', err); - }); - } else if (isMobile) { - try { - browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } catch (err) { - console.error('[useBrowserVoice] Failed to restart listening:', err); - } - } else { - browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } + startCurrentSTT(language); } else { // In non-continuous mode, return to idle after AI responds isActiveRef.current = false; @@ -547,19 +589,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { // Only restart listening if conversation mode is enabled if (conversationMode) { setStatus('listening'); - if (sttProvider === 'server') { - audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!).catch((err) => { - console.error('[useBrowserVoice] Failed to restart server STT after speech error:', err); - }); - } else if (isMobile) { - try { - browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } catch (restartErr) { - console.error('[useBrowserVoice] Failed to restart listening after speech error:', restartErr); - } - } else { - browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } + startCurrentSTT(language); } else { // In non-continuous mode, return to idle after error isActiveRef.current = false; @@ -583,7 +613,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { setStatus('error'); processingMessageRef.current = false; } - }, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, openaiCompatibleTtsModel, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, sttProvider]); + }, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, openaiCompatibleVoice, openaiCompatibleUrl, openaiCompatibleTtsModel, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode, startCurrentSTT]); // Handle speech recognition result const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => { @@ -591,6 +621,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { const normalized = text.trim(); if (!isFinal || !normalized) return; + console.log('[useBrowserVoice] Speech result:', normalized); pendingFinalTranscriptRef.current = normalized; if (finalTranscriptTimerRef.current) { @@ -626,13 +657,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { pendingResumeOnVisibleRef.current = false; setStatus('listening'); try { - if (sttProvider === 'server') { - void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } else if (isMobile) { - browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } else { - browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } + startCurrentSTT(language); } catch (err) { const errorMsg = err instanceof Error ? err.message : 'Failed to resume voice'; setError(errorMsg); @@ -644,7 +669,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { return () => { document.removeEventListener('visibilitychange', handleVisibilityChange); }; - }, [conversationMode, isMobile, language, sttProvider]); + }, [conversationMode, isMobile, language, sttProvider, startCurrentSTT]); useEffect(() => { if (typeof navigator === 'undefined') { @@ -679,17 +704,10 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { } try { - if (sttProvider === 'server') { - audioStreamService.stopListening(); - void audioStreamService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } else { - browserVoiceService.stopListening(); - if (isMobile) { - browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } else { - void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!); - } - } + browserVoiceService.stopListening(); + audioStreamService.stopListening(); + wasmSttService.stopListening(); + startCurrentSTT(language); } catch (err) { const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.'; setError(errorMsg); @@ -708,7 +726,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { deviceChangeRestartTimerRef.current = null; } }; - }, [isMobile, language, status, sttProvider]); + }, [isMobile, language, status, sttProvider, startCurrentSTT]); // Update the ref when handleSpeechResult changes useEffect(() => { @@ -720,8 +738,8 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { if (!isSupported) { return false; } - if (sttProvider === 'server') { - // getUserMedia permission is requested on startListening; nothing to prepare + if (sttProvider === 'server' || sttProvider === 'wasm') { + // Permission is requested on startListening; nothing to pre-prepare return true; } try { @@ -750,6 +768,12 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { isActiveRef.current = true; lastTranscriptRef.current = ''; + consecutiveRecoveryRetriesRef.current = 0; + isFinalizingRef.current = false; + if (recoveryTimerRef.current !== null) { + clearTimeout(recoveryTimerRef.current); + recoveryTimerRef.current = null; + } setError(null); setStatus('listening'); @@ -774,6 +798,40 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { return; } + if (sttProvider === 'wasm') { + // WASM STT: ensure model is loaded then start recording + const modelStatus = wasmSttService.getModelStatus(); + console.log('[useBrowserVoice] WASM model status:', modelStatus.state); + if (modelStatus.state !== 'ready') { + try { + setStatus('processing'); + await wasmSttService.loadModel(wasmSttModel); + } catch (err) { + const errorMsg = err instanceof Error ? err.message : 'Failed to load Whisper model'; + console.error('[useBrowserVoice] WASM model load error:', errorMsg); + setError(errorMsg); + setStatus('error'); + isActiveRef.current = false; + return; + } + } + wasmSttService.configure({ + silenceThresholdDb: sttSilenceThresholdDb, + silenceHoldMs: sttSilenceHoldMs, + }); + try { + await wasmSttService.startListening(language, handleSpeechResult, handleSpeechError); + console.log('[useBrowserVoice] WASM listening started'); + } catch (err) { + const errorMsg = err instanceof Error ? err.message : 'Failed to start voice'; + console.error('[useBrowserVoice] WASM STT start error:', errorMsg); + setError(errorMsg); + setStatus('error'); + isActiveRef.current = false; + } + return; + } + // Browser STT // On mobile, use sync path to ensure SpeechRecognition.start() is called // within the same user gesture context (required by iOS Safari) @@ -812,17 +870,23 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { isActiveRef.current = false; } } - }, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]); + }, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio, sttProvider, sttServerUrl, sttModel, wasmSttModel, sttLanguage, sttSilenceThresholdDb, sttSilenceHoldMs]); // Stop voice mode const stopVoice = useCallback(() => { isActiveRef.current = false; processingMessageRef.current = false; pendingResumeOnVisibleRef.current = false; + consecutiveRecoveryRetriesRef.current = 0; + isFinalizingRef.current = false; if (deviceChangeRestartTimerRef.current) { clearTimeout(deviceChangeRestartTimerRef.current); deviceChangeRestartTimerRef.current = null; } + if (recoveryTimerRef.current !== null) { + clearTimeout(recoveryTimerRef.current); + recoveryTimerRef.current = null; + } pendingFinalTranscriptRef.current = ''; if (finalTranscriptTimerRef.current) { clearTimeout(finalTranscriptTimerRef.current); @@ -830,6 +894,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { } browserVoiceService.stopListening(); audioStreamService.stopListening(); + wasmSttService.stopListening(); browserVoiceService.cancelSpeech(); stopServerTTS(); // Also stop server TTS if playing stopSayTTS(); // Also stop Say TTS if playing @@ -842,6 +907,7 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { return; } + isFinalizingRef.current = true; pendingResumeOnVisibleRef.current = false; if (deviceChangeRestartTimerRef.current) { clearTimeout(deviceChangeRestartTimerRef.current); @@ -869,6 +935,28 @@ export function useBrowserVoice(): UseBrowserVoiceReturn { return; } + if (sttProvider === 'wasm') { + // Inference runs in a Web Worker — no main-thread freeze. + void wasmSttService.finishListening().then(() => { + window.setTimeout(() => { + if (!isActiveRef.current) { + return; + } + if (pendingFinalTranscriptRef.current || finalTranscriptTimerRef.current) { + return; + } + if (processingMessageRef.current) { + return; + } + isActiveRef.current = false; + processingMessageRef.current = false; + setStatus('idle'); + isFinalizingRef.current = false; + }, FINAL_TRANSCRIPT_SETTLE_MS + 200); + }); + return; + } + browserVoiceService.stopListening(); window.setTimeout(() => { if (!isActiveRef.current) { diff --git a/packages/ui/src/lib/desktop.ts b/packages/ui/src/lib/desktop.ts index 72255eb0..c11d6751 100644 --- a/packages/ui/src/lib/desktop.ts +++ b/packages/ui/src/lib/desktop.ts @@ -166,9 +166,10 @@ export type DesktopSettings = { responseStyleEnabled?: boolean; responseStylePreset?: 'concise' | 'detailed' | 'mentor' | 'pushback' | 'noFiller' | 'matchEnergy' | 'warmPeer' | 'custom'; responseStyleCustomInstructions?: string; - sttProvider?: 'browser' | 'server'; + sttProvider?: 'browser' | 'server' | 'wasm'; sttServerUrl?: string; sttModel?: string; + wasmSttModel?: string; sttLanguage?: string; sttSilenceThresholdDb?: number; sttSilenceHoldMs?: number; diff --git a/packages/ui/src/lib/i18n/messages/en.settings.ts b/packages/ui/src/lib/i18n/messages/en.settings.ts index 49724cbc..53ff254a 100644 --- a/packages/ui/src/lib/i18n/messages/en.settings.ts +++ b/packages/ui/src/lib/i18n/messages/en.settings.ts @@ -1301,6 +1301,14 @@ export const settingsDict = { 'settings.voice.page.provider.custom': 'Custom', 'settings.voice.page.provider.say': 'Say', 'settings.voice.page.provider.server': 'Server', + 'settings.voice.page.provider.wasm': 'Local', + 'settings.voice.page.stt.wasmModel': 'Whisper Model', + 'settings.voice.page.stt.wasmLoaded': 'Model loaded, ready', + 'settings.voice.page.stt.wasmDownloading': 'Downloading model...', + 'settings.voice.page.stt.wasmLoading': 'Loading model...', + 'settings.voice.page.stt.wasmNotLoaded': 'Model will load on first voice use', + 'settings.voice.page.stt.wasmDownload': 'Load', + 'settings.voice.page.stt.wasmRetry': 'Retry', 'settings.voice.page.tooltip.browser': 'Free, offline, limited mobile support.', 'settings.voice.page.tooltip.openai': 'High quality, mobile ready, needs API key.', 'settings.voice.page.tooltip.custom': 'OpenAI-compatible server (for example Kokoro).', diff --git a/packages/ui/src/lib/i18n/messages/es.settings.ts b/packages/ui/src/lib/i18n/messages/es.settings.ts index 188ff9d9..e5dc9e72 100644 --- a/packages/ui/src/lib/i18n/messages/es.settings.ts +++ b/packages/ui/src/lib/i18n/messages/es.settings.ts @@ -1301,6 +1301,14 @@ export const settingsDict = { "settings.voice.page.provider.custom": "Personalizado", "settings.voice.page.provider.say": "Decir", "settings.voice.page.provider.server": "Servidor", + "settings.voice.page.provider.wasm": "Local", + "settings.voice.page.stt.wasmModel": "Modelo Whisper", + "settings.voice.page.stt.wasmLoaded": "Modelo cargado", + "settings.voice.page.stt.wasmDownloading": "Descargando modelo...", + "settings.voice.page.stt.wasmLoading": "Cargando modelo...", + "settings.voice.page.stt.wasmNotLoaded": "El modelo se descargará en el primer uso de voz", + "settings.voice.page.stt.wasmDownload": "Descargar", + "settings.voice.page.stt.wasmRetry": "Reintentar", "settings.voice.page.tooltip.browser": "Gratuito, offline, soporte limitado en móvil.", "settings.voice.page.tooltip.openai": "Calidad alta, compatible con móviles, requiere clave de API.", "settings.voice.page.tooltip.custom": "Servidor compatible con OpenAI (por ejemplo Kokoro).", diff --git a/packages/ui/src/lib/i18n/messages/ko.settings.ts b/packages/ui/src/lib/i18n/messages/ko.settings.ts index f87baebd..731ce627 100644 --- a/packages/ui/src/lib/i18n/messages/ko.settings.ts +++ b/packages/ui/src/lib/i18n/messages/ko.settings.ts @@ -1301,6 +1301,14 @@ export const settingsDict = { 'settings.voice.page.provider.custom': '사용자 정의', 'settings.voice.page.provider.say': 'Say', 'settings.voice.page.provider.server': '서버', + 'settings.voice.page.provider.wasm': '로컬', + 'settings.voice.page.stt.wasmModel': 'Whisper 모델', + 'settings.voice.page.stt.wasmLoaded': '모델 로드됨', + 'settings.voice.page.stt.wasmDownloading': '모델 다운로드 중...', + 'settings.voice.page.stt.wasmLoading': '모델 로드 중...', + 'settings.voice.page.stt.wasmNotLoaded': '첫 음성 사용 시 모델이 다운로드됩니다', + 'settings.voice.page.stt.wasmDownload': '다운로드', + 'settings.voice.page.stt.wasmRetry': '재시도', 'settings.voice.page.tooltip.browser': '무료, 오프라인, 모바일 지원 제한.', 'settings.voice.page.tooltip.openai': '고품질, 모바일 지원, API key 필요.', 'settings.voice.page.tooltip.custom': 'OpenAI 호환 서버(예: Kokoro).', diff --git a/packages/ui/src/lib/i18n/messages/pl.settings.ts b/packages/ui/src/lib/i18n/messages/pl.settings.ts index 5e48e5b2..585ccecc 100644 --- a/packages/ui/src/lib/i18n/messages/pl.settings.ts +++ b/packages/ui/src/lib/i18n/messages/pl.settings.ts @@ -1527,6 +1527,14 @@ export const settingsDict = { 'settings.voice.page.provider.custom': 'Własny', 'settings.voice.page.provider.say': 'Say', 'settings.voice.page.provider.server': 'Serwer', + 'settings.voice.page.provider.wasm': 'Lokalny', + 'settings.voice.page.stt.wasmModel': 'Model Whisper', + 'settings.voice.page.stt.wasmLoaded': 'Model załadowany', + 'settings.voice.page.stt.wasmDownloading': 'Pobieranie modelu...', + 'settings.voice.page.stt.wasmLoading': 'Ładowanie modelu...', + 'settings.voice.page.stt.wasmNotLoaded': 'Model zostanie pobrany przy pierwszym użyciu', + 'settings.voice.page.stt.wasmDownload': 'Pobierz', + 'settings.voice.page.stt.wasmRetry': 'Spróbuj ponownie', 'settings.voice.page.section.playbackAndSummary': 'Odtwarzanie i podsumowanie', 'settings.voice.page.section.speechRecognition': 'Rozpoznawanie mowy', 'settings.voice.page.section.voiceSetup': 'Konfiguracja głosu', diff --git a/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts b/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts index 4157aab4..f3f52b2e 100644 --- a/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts +++ b/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts @@ -1301,6 +1301,14 @@ export const settingsDict = { "settings.voice.page.provider.custom": "Personalizado", "settings.voice.page.provider.say": "Falar", "settings.voice.page.provider.server": "Servidor", + "settings.voice.page.provider.wasm": "Local", + "settings.voice.page.stt.wasmModel": "Modelo Whisper", + "settings.voice.page.stt.wasmLoaded": "Modelo carregado", + "settings.voice.page.stt.wasmDownloading": "Baixando modelo...", + "settings.voice.page.stt.wasmLoading": "Carregando modelo...", + "settings.voice.page.stt.wasmNotLoaded": "O modelo será baixado no primeiro uso", + "settings.voice.page.stt.wasmDownload": "Baixar", + "settings.voice.page.stt.wasmRetry": "Tentar novamente", "settings.voice.page.tooltip.browser": "Gratuito, offline, suporte limitado em dispositivos móveis.", "settings.voice.page.tooltip.openai": "Alta qualidade, compatível com dispositivos móveis, exige chave de API.", "settings.voice.page.tooltip.custom": "Servidor compatível com OpenAI (por exemplo Kokoro).", diff --git a/packages/ui/src/lib/i18n/messages/uk.settings.ts b/packages/ui/src/lib/i18n/messages/uk.settings.ts index 8e3a7669..8d808044 100644 --- a/packages/ui/src/lib/i18n/messages/uk.settings.ts +++ b/packages/ui/src/lib/i18n/messages/uk.settings.ts @@ -1301,6 +1301,14 @@ export const settingsDict = { "settings.voice.page.provider.custom": "Власний", "settings.voice.page.provider.say": "Say", "settings.voice.page.provider.server": "Сервер", + "settings.voice.page.provider.wasm": "Локально", + "settings.voice.page.stt.wasmModel": "Модель Whisper", + "settings.voice.page.stt.wasmLoaded": "Модель завантажено", + "settings.voice.page.stt.wasmDownloading": "Завантаження моделі...", + "settings.voice.page.stt.wasmLoading": "Завантаження моделі...", + "settings.voice.page.stt.wasmNotLoaded": "Модель завантажиться при першому використанні", + "settings.voice.page.stt.wasmDownload": "Завантажити", + "settings.voice.page.stt.wasmRetry": "Спробувати знову", "settings.voice.page.tooltip.browser": "Безкоштовна, офлайн, обмежена мобільна підтримка.", "settings.voice.page.tooltip.openai": "Висока якість, готово для мобільних пристроїв, потрібен API ключ.", "settings.voice.page.tooltip.custom": "OpenAI-сумісний сервер, наприклад Kokoro.", diff --git a/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts b/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts index 4d94a006..ee0ca6c9 100644 --- a/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts +++ b/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts @@ -1301,6 +1301,14 @@ export const settingsDict = { 'settings.voice.page.provider.custom': '自定义', 'settings.voice.page.provider.say': 'Say', 'settings.voice.page.provider.server': '服务器', + 'settings.voice.page.provider.wasm': '本地', + 'settings.voice.page.stt.wasmModel': 'Whisper 模型', + 'settings.voice.page.stt.wasmLoaded': '模型已加载', + 'settings.voice.page.stt.wasmDownloading': '下载模型中...', + 'settings.voice.page.stt.wasmLoading': '加载模型中...', + 'settings.voice.page.stt.wasmNotLoaded': '模型将在首次使用时下载', + 'settings.voice.page.stt.wasmDownload': '下载', + 'settings.voice.page.stt.wasmRetry': '重试', 'settings.voice.page.tooltip.browser': '免费、离线，移动端支持有限。', 'settings.voice.page.tooltip.openai': '质量高，移动端可用，需要 API Key。', 'settings.voice.page.tooltip.custom': 'OpenAI 兼容服务器（例如 Kokoro）。', diff --git a/packages/ui/src/lib/voice/browserVoiceService.ts b/packages/ui/src/lib/voice/browserVoiceService.ts index 8aaa9b26..439d946b 100644 --- a/packages/ui/src/lib/voice/browserVoiceService.ts +++ b/packages/ui/src/lib/voice/browserVoiceService.ts @@ -282,8 +282,10 @@ class BrowserVoiceService { const errorMessage = this.getErrorMessage(event.error); this.onErrorCallback?.(errorMessage); - // Don't restart on certain errors - if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { + // Don't restart on fatal / unrecoverable errors. + // "network" in Electron/Chromium means Google's speech servers are unreachable; + // auto-restarting immediately creates an infinite error → end → start → error loop. + if (event.error === 'not-allowed' || event.error === 'service-not-allowed' || event.error === 'network') { this.restartOnEnd = false; this.isListening = false; } @@ -292,12 +294,13 @@ class BrowserVoiceService { this.recognition.onend = () => { this.isListening = false; - // Auto-restart if still supposed to be listening and not speaking + // Auto-restart if still supposed to be listening and not speaking. + // Only restart when we have a valid recognition instance and restartOnEnd is set. if (this.restartOnEnd && this.recognition && !this.isSpeaking) { try { this.recognition.start(); } catch { - // Ignore restart errors + // Ignore restart errors — onerror / onend will fire if it's fatal. } } }; diff --git a/packages/ui/src/lib/voice/wasmSttService.ts b/packages/ui/src/lib/voice/wasmSttService.ts new file mode 100644 index 00000000..7a2f6111 --- /dev/null +++ b/packages/ui/src/lib/voice/wasmSttService.ts @@ -0,0 +1,556 @@ +/** + * WASM Speech-to-Text Service + * + * Local Whisper transcription via Transformers.js (ONNX Runtime Web). + * Captures microphone audio, detects utterance boundaries via silence-based + * VAD, then transcribes each utterance locally — no cloud API required. + * + * Works in Electron and all modern browsers that support Web Audio API. + * First use downloads a Whisper model (~40–166 MB, cached). + */ + +export type WasmModelStatus = + | { state: 'unloaded' } + | { state: 'downloading'; progress: number } + | { state: 'loading' } + | { state: 'ready' } + | { state: 'error'; error: string }; + +export interface WasmModelInfo { + id: string; + name: string; + size: string; + languages: string; + description: string; +} + +export const WASM_MODELS: WasmModelInfo[] = [ + { + id: 'Xenova/whisper-tiny.en', + name: 'Whisper Tiny (EN)', + size: '~39 MB', + languages: 'English', + description: 'Fastest, lowest accuracy. Good for quick dictation.', + }, + { + id: 'Xenova/whisper-base.en', + name: 'Whisper Base (EN)', + size: '~73 MB', + languages: 'English', + description: 'Balanced speed and accuracy. Default for English.', + }, + { + id: 'Xenova/whisper-small.en', + name: 'Whisper Small (EN)', + size: '~166 MB', + languages: 'English', + description: 'Higher accuracy, slower. Best for noisy environments.', + }, +]; + +export type SpeechResultCallback = (text: string, isFinal: boolean) => void; +export type ErrorCallback = (error: string) => void; + +const VAD_POLL_MS = 80; +const MIN_UTTERANCE_MS = 300; +const WHISPER_SAMPLE_RATE = 16000; + +interface WasmSttConfig { + silenceThresholdDb?: number; + silenceHoldMs?: number; +} + +class WasmSttService { + private transcriber: unknown = null; + private worker: Worker | null = null; + private modelStatus: WasmModelStatus = { state: 'unloaded' }; + private currentModelId: string | null = null; + + private stream: MediaStream | null = null; + private mediaRecorder: MediaRecorder | null = null; + private audioContext: AudioContext | null = null; + private analyser: AnalyserNode | null = null; + private vadTimer: ReturnType | null = null; + private chunks: Blob[] = []; + private recordingStartMs = 0; + private isActive = false; + private isSpeaking = false; + private silenceSince: number | null = null; + private onResult: SpeechResultCallback | null = null; + private onError: ErrorCallback | null = null; + private finishResolver: (() => void) | null = null; + private lang = 'en'; + + private cfg: Required = { + silenceThresholdDb: -45, + silenceHoldMs: 1500, + }; + + public onModelStatusChange: ((status: WasmModelStatus) => void) | null = null; + + configure(config: WasmSttConfig): void { + this.cfg = { ...this.cfg, ...config }; + } + + isSupported(): boolean { + return ( + typeof window !== 'undefined' && + typeof navigator !== 'undefined' && + typeof navigator.mediaDevices?.getUserMedia === 'function' && + typeof window.MediaRecorder !== 'undefined' && + typeof window.AudioContext !== 'undefined' + ); + } + + getModelStatus(): WasmModelStatus { + return this.modelStatus; + } + + getCurrentModelId(): string | null { + return this.currentModelId; + } + + private setModelStatus(status: WasmModelStatus): void { + this.modelStatus = status; + this.onModelStatusChange?.(status); + } + + async loadModel(modelId: string): Promise { + if (this.currentModelId === modelId && this.modelStatus.state === 'ready') { + return; + } + + if (this.modelStatus.state === 'downloading' || this.modelStatus.state === 'loading') { + return; + } + + this._terminateWorker(); + this.transcriber = null; + + this.setModelStatus({ state: 'downloading', progress: 0 }); + this.currentModelId = modelId; + + // Try Web Worker first — inference off main thread = no UI freeze. + try { + const WasmWorkerMod = await import('./wasmSttWorker?worker'); + const WasmWorker = WasmWorkerMod.default as new () => Worker; + this.worker = new WasmWorker(); + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Worker init timed out')), 10000); + + this.worker!.onmessage = (e: MessageEvent) => { + const data = e.data as { type: string; progress?: number; error?: string; text?: string }; + if (data.type === 'progress') { + this.setModelStatus({ state: 'downloading', progress: data.progress ?? 0 }); + } else if (data.type === 'loaded') { + clearTimeout(timer); + resolve(); + } else if (data.type === 'error') { + clearTimeout(timer); + reject(new Error(data.error ?? 'Worker load failed')); + } + }; + + this.worker!.onerror = (err) => { + clearTimeout(timer); + reject(new Error(err.message || 'Worker error')); + }; + + this.worker!.postMessage({ type: 'load', modelId }); + }); + + this.setModelStatus({ state: 'ready' }); + return; + } catch (err) { + console.warn('[WasmStt] Worker failed, using main-thread:', err instanceof Error ? err.message : err); + this._terminateWorker(); + } + + // Fallback: main-thread pipeline (causes brief UI freeze during inference). + try { + const { pipeline, env } = await import('@xenova/transformers'); + env.backends.onnx.wasm.numThreads = 1; + env.allowLocalModels = false; + + const fileDoneBytes = new Map(); + let totalDone = 0; + let totalEstimate = 0; + + this.transcriber = await pipeline('automatic-speech-recognition', modelId, { + progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => { + if (info.status === 'progress' && info.file) { + const prevDone = fileDoneBytes.get(info.file) ?? 0; + const currentDone = info.loaded ?? 0; + const delta = Math.max(0, currentDone - prevDone); + fileDoneBytes.set(info.file, currentDone); + totalDone += delta; + if (info.total && info.total > totalEstimate) totalEstimate = info.total; + const effectiveTotal = Math.max(totalEstimate, totalDone); + const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0; + this.setModelStatus({ state: 'downloading', progress: pct }); + } + }, + }); + + this.setModelStatus({ state: 'ready' }); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Unknown error loading model'; + this.setModelStatus({ state: 'error', error: msg }); + this.transcriber = null; + this.currentModelId = null; + throw err; + } + } + + private _terminateWorker(): void { + if (this.worker) { + this.worker.terminate(); + this.worker = null; + } + } + + async unloadModel(): Promise { + this._terminateWorker(); + this.transcriber = null; + this.currentModelId = null; + this.setModelStatus({ state: 'unloaded' }); + } + + async startListening( + lang: string, + onResult: SpeechResultCallback, + onError?: ErrorCallback, + ): Promise { + if (this.isActive) { + this.stopListening(); + } + + if (!this.transcriber && !this.worker) { + onError?.('Whisper model not loaded. Select a model in Voice Settings first.'); + return; + } + + this.lang = lang; + this.onResult = onResult; + this.onError = onError ?? null; + this.isActive = true; + + try { + this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + } catch (err) { + this.isActive = false; + const msg = err instanceof Error ? err.message : 'Microphone access denied'; + onError?.(msg); + return; + } + + this._setupAudioContext(); + this._startRecorder(); + this._startVAD(); + } + + stopListening(): void { + this._stopVAD(); + if (this.mediaRecorder && this.mediaRecorder.state === 'recording') { + try { this.mediaRecorder.stop(); } catch { /* ignore */ } + } + this._cleanupAfterStop(true); + } + + async finishListening(): Promise { + if (!this.isActive) return; + + this._stopVAD(); + this.isSpeaking = false; + this.silenceSince = null; + + if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive') { + this._cleanupAfterStop(true); + return; + } + + await new Promise((resolve) => { + this.finishResolver = resolve; + this._finaliseUtterance(false); + }); + + this._cleanupAfterStop(true); + } + + getIsListening(): boolean { + return this.isActive; + } + + // ── Audio capture ──────────────────────────────────────────────────── + + private _setupAudioContext(): void { + if (!this.stream) return; + const AudioContextClass = window.AudioContext ?? (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext; + this.audioContext = new AudioContextClass(); + const source = this.audioContext.createMediaStreamSource(this.stream); + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + source.connect(this.analyser); + } + + private _teardownAudioContext(): void { + try { this.audioContext?.close(); } catch { /* ignore */ } + this.audioContext = null; + this.analyser = null; + } + + private _startRecorder(): void { + if (!this.stream) return; + const mimeType = this._pickMimeType(); + const options: MediaRecorderOptions = {}; + if (mimeType && MediaRecorder.isTypeSupported(mimeType)) { + options.mimeType = mimeType; + } + this.mediaRecorder = new MediaRecorder(this.stream, options); + this.chunks = []; + this.recordingStartMs = Date.now(); + + this.mediaRecorder.ondataavailable = (e) => { + if (e.data && e.data.size > 0) { + this.chunks.push(e.data); + } + }; + + this.mediaRecorder.onstop = () => { + const blobs = this.chunks.splice(0); + const durationMs = Date.now() - this.recordingStartMs; + if (blobs.length === 0 || durationMs < MIN_UTTERANCE_MS) { + this.finishResolver?.(); + this.finishResolver = null; + return; + } + const mType = blobs[0].type || mimeType || 'audio/webm'; + const blob = new Blob(blobs, { type: mType }); + void this._transcribe(blob).finally(() => { + this.finishResolver?.(); + this.finishResolver = null; + }); + }; + + this.mediaRecorder.start(250); + } + + private _releaseStream(): void { + if (this.stream) { + this.stream.getTracks().forEach((t) => t.stop()); + this.stream = null; + } + } + + // ── VAD ────────────────────────────────────────────────────────────── + + private _startVAD(): void { + this._stopVAD(); + this.silenceSince = null; + this.isSpeaking = false; + + this.vadTimer = setInterval(() => { + if (!this.isActive || !this.analyser) return; + const db = this._getRmsDb(); + const isSilent = db < this.cfg.silenceThresholdDb; + + if (!isSilent) { + this.silenceSince = null; + if (!this.isSpeaking) { + this.isSpeaking = true; + if (this.mediaRecorder?.state === 'recording') { + this.recordingStartMs = Date.now(); + } + } + } else { + if (this.isSpeaking) { + if (this.silenceSince === null) { + this.silenceSince = Date.now(); + } else if (Date.now() - this.silenceSince >= this.cfg.silenceHoldMs) { + this.isSpeaking = false; + this.silenceSince = null; + this._finaliseUtterance(true); + } + } + } + }, VAD_POLL_MS); + } + + private _stopVAD(): void { + if (this.vadTimer !== null) { + clearInterval(this.vadTimer); + this.vadTimer = null; + } + } + + private _cleanupAfterStop(clearChunks: boolean): void { + const pendingResolver = this.finishResolver; + this.isActive = false; + this.finishResolver = null; + this.mediaRecorder = null; + this._teardownAudioContext(); + this._releaseStream(); + if (clearChunks) this.chunks = []; + this.isSpeaking = false; + this.silenceSince = null; + this.onResult = null; + this.onError = null; + pendingResolver?.(); + } + + private _finaliseUtterance(restart: boolean): void { + if (!this.isActive) return; + if (this.mediaRecorder && this.mediaRecorder.state === 'recording') { + this.mediaRecorder.stop(); + } + if (!restart) return; + setTimeout(() => { + if (this.isActive && this.stream) { + this._startRecorder(); + } + }, 100); + } + + private _getRmsDb(): number { + if (!this.analyser) return -Infinity; + const buf = new Float32Array(this.analyser.fftSize); + this.analyser.getFloatTimeDomainData(buf); + let sumSq = 0; + for (const s of buf) sumSq += s * s; + const rms = Math.sqrt(sumSq / buf.length); + return rms === 0 ? -Infinity : 20 * Math.log10(rms); + } + + // ── Transcription ──────────────────────────────────────────────────── + + private async _transcribe(blob: Blob): Promise { + if (!this.onResult) return; + if (!this.transcriber && !this.worker) { + this.onError?.('Model not loaded'); + return; + } + + try { + const audioData = await this._decodeToFloat32(blob); + if (!audioData || audioData.length === 0) { + this.onError?.(`Failed to decode audio (${blob.size} bytes)`); + return; + } + + const langHint = this._resolveLanguageHint(); + + // Prefer worker (non-blocking); fall back to main-thread pipeline. + const transcript = this.worker + ? await this._transcribeViaWorker(audioData, langHint) + : await this._transcribeMainThread(audioData, langHint); + + if (transcript) { + this.onResult(transcript, true); + } + } catch (err) { + if (!this.isActive) return; + const msg = err instanceof Error ? err.message : 'Local transcription failed'; + this.onError?.(msg); + } + } + + private _transcribeViaWorker(audioData: Float32Array, langHint: string | undefined): Promise { + return new Promise((resolve, reject) => { + if (!this.worker) return reject(new Error('Worker gone')); + + const onMessage = (e: MessageEvent) => { + const data = e.data as { type: string; error?: string; transcript?: string; text?: string }; + if (data.type === 'result') { + this.worker!.removeEventListener('message', onMessage); + resolve(data.transcript ?? ''); + } else if (data.type === 'log') { + console.log('[WasmStt Worker]', data.text); + } else if (data.type === 'error') { + this.worker!.removeEventListener('message', onMessage); + reject(new Error(data.error ?? 'Transcription failed')); + } + }; + + this.worker.addEventListener('message', onMessage); + this.worker.postMessage( + { type: 'transcribe', audio: audioData.buffer, language: langHint }, + [audioData.buffer], + ); + + setTimeout(() => { + this.worker?.removeEventListener('message', onMessage); + reject(new Error('Transcription timed out')); + }, 30000); + }); + } + + private async _transcribeMainThread(audioData: Float32Array, langHint: string | undefined): Promise { + const pipelineFn = this.transcriber as ( + input: Float32Array, + options?: Record, + ) => Promise<{ text: string }>; + + const result = await pipelineFn(audioData, { + task: 'transcribe', + ...(langHint ? { language: langHint } : {}), + }); + + return (result?.text ?? '').trim(); + } + + private async _decodeToFloat32(blob: Blob): Promise { + if (!this.audioContext) return null; + const arrayBuffer = await blob.arrayBuffer(); + let audioBuffer: AudioBuffer; + try { + audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer); + } catch { + return null; + } + + const origRate = audioBuffer.sampleRate; + const origData = audioBuffer.getChannelData(0); + const targetRate = WHISPER_SAMPLE_RATE; + + if (origRate === targetRate) { + return new Float32Array(origData); + } + + const ratio = origRate / targetRate; + const newLength = Math.ceil(origData.length / ratio); + const result = new Float32Array(newLength); + for (let i = 0; i < newLength; i++) { + const origIdx = i * ratio; + const idx0 = Math.floor(origIdx); + const idx1 = Math.min(idx0 + 1, origData.length - 1); + const frac = origIdx - idx0; + result[i] = origData[idx0] * (1 - frac) + origData[idx1] * frac; + } + return result; + } + + private _resolveLanguageHint(): string | undefined { + if (this.lang && this.lang !== 'auto') { + return this.lang.split('-')[0]; + } + return undefined; + } + + private _pickMimeType(): string { + const candidates = [ + 'audio/webm;codecs=opus', + 'audio/webm', + 'audio/ogg;codecs=opus', + 'audio/ogg', + 'audio/mp4', + ]; + if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported) { + return candidates.find((t) => MediaRecorder.isTypeSupported(t)) ?? ''; + } + return ''; + } +} + +export const wasmSttService = new WasmSttService(); +export { WasmSttService }; diff --git a/packages/ui/src/lib/voice/wasmSttWorker.ts b/packages/ui/src/lib/voice/wasmSttWorker.ts new file mode 100644 index 00000000..cb5b48ad --- /dev/null +++ b/packages/ui/src/lib/voice/wasmSttWorker.ts @@ -0,0 +1,92 @@ +/** + * Web Worker for off-main-thread Whisper transcription. + * + * Receives `{ type: 'load', modelId }` to load a model, then + * `{ type: 'transcribe', audio: Float32Array (transferred buffer), language? }` + * to run inference. Posts progress, results, and errors back. + */ + +import { pipeline, env } from '@xenova/transformers'; + +let transcriber: unknown = null; + +self.onmessage = async (e: MessageEvent) => { + const { type } = e.data as { type: string }; + + if (type === 'load') { + const { modelId } = e.data as { modelId: string }; + try { + env.backends.onnx.wasm.numThreads = 1; + + const fileDoneBytes = new Map(); + let totalDone = 0; + let totalEstimate = 0; + + transcriber = await pipeline('automatic-speech-recognition', modelId, { + progress_callback: (info: { status?: string; file?: string; loaded?: number; total?: number }) => { + if (info.status === 'progress' && info.file) { + const prevDone = fileDoneBytes.get(info.file) ?? 0; + const currentDone = info.loaded ?? 0; + const delta = Math.max(0, currentDone - prevDone); + fileDoneBytes.set(info.file, currentDone); + totalDone += delta; + + if (info.total && info.total > totalEstimate) { + totalEstimate = info.total; + } + + const effectiveTotal = Math.max(totalEstimate, totalDone); + const pct = effectiveTotal > 0 ? Math.min(100, Math.round((totalDone / effectiveTotal) * 100)) : 0; + self.postMessage({ type: 'progress', progress: pct }); + } + }, + }); + + self.postMessage({ type: 'loaded' }); + } catch (err) { + self.postMessage({ + type: 'error', + error: err instanceof Error ? err.message : 'Failed to load model', + }); + } + } else if (type === 'transcribe') { + if (!transcriber) { + self.postMessage({ type: 'error', error: 'Model not loaded', seq: (e.data as { seq?: number }).seq }); + return; + } + + const { audio, language, seq } = e.data as { audio: ArrayBuffer; language?: string; seq?: number }; + + try { + const samples = new Float32Array(audio); + if (samples.length === 0) { + self.postMessage({ type: 'error', error: 'Empty audio received', seq }); + return; + } + + self.postMessage({ type: 'log', text: `Transcribing ${samples.length} samples (${(samples.length / 16000).toFixed(1)}s)` }); + + const pipelineFn = transcriber as ( + input: Float32Array, + options?: Record, + ) => Promise<{ text: string }>; + + const result = await pipelineFn(samples, { + task: 'transcribe', + ...(language ? { language } : {}), + }); + + self.postMessage({ + type: 'result', + transcript: (result?.text ?? '').trim(), + seq, + }); + } catch (err) { + self.postMessage({ + type: 'error', + error: err instanceof Error ? err.message : 'Transcription failed', + seq, + }); + } + } +}; diff --git a/packages/ui/src/stores/useConfigStore.ts b/packages/ui/src/stores/useConfigStore.ts index 48d5dc2f..7b847904 100644 --- a/packages/ui/src/stores/useConfigStore.ts +++ b/packages/ui/src/stores/useConfigStore.ts @@ -50,8 +50,9 @@ interface OpenChamberDefaults { defaultFileViewerPreview?: boolean; zenModel?: string; messageStreamTransport?: 'auto' | 'ws' | 'sse'; - sttProvider?: 'browser' | 'server'; + sttProvider?: 'browser' | 'server' | 'wasm'; sttServerUrl?: string; + wasmSttModel?: string; sttModel?: string; sttLanguage?: string; sttSilenceThresholdDb?: number; @@ -77,7 +78,7 @@ const fetchOpenChamberDefaults = async (): Promise => { data?.messageStreamTransport === 'ws' || data?.messageStreamTransport === 'sse' || data?.messageStreamTransport === 'auto' ? data.messageStreamTransport : undefined; - const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' ? data.sttProvider : undefined; + const sttProvider = data?.sttProvider === 'browser' || data?.sttProvider === 'server' || data?.sttProvider === 'wasm' ? data.sttProvider : undefined; const sttServerUrl = typeof data?.sttServerUrl === 'string' ? data.sttServerUrl.trim() : undefined; const sttModel = typeof data?.sttModel === 'string' ? data.sttModel.trim() : undefined; const sttLanguage = typeof data?.sttLanguage === 'string' ? data.sttLanguage.trim() : undefined; @@ -552,9 +553,10 @@ interface ConfigStore { openaiCompatibleVoice: string; openaiCompatibleTtsModel: string; // STT (speech-to-text) settings - sttProvider: 'browser' | 'server'; + sttProvider: 'browser' | 'server' | 'wasm'; sttServerUrl: string; sttModel: string; + wasmSttModel: string; sttLanguage: string; sttSilenceThresholdDb: number; sttSilenceHoldMs: number; @@ -576,9 +578,10 @@ interface ConfigStore { setOpenaiCompatibleUrl: (url: string) => void; setOpenaiCompatibleVoice: (voice: string) => void; setOpenaiCompatibleTtsModel: (model: string) => void; - setSttProvider: (provider: 'browser' | 'server') => void; + setSttProvider: (provider: 'browser' | 'server' | 'wasm') => void; setSttServerUrl: (url: string) => void; setSttModel: (model: string) => void; + setWasmSttModel: (model: string) => void; setSttLanguage: (lang: string) => void; setSttSilenceThresholdDb: (db: number) => void; setSttSilenceHoldMs: (ms: number) => void; @@ -761,11 +764,15 @@ export const useConfigStore = create()( } return 'kokoro'; })(), - // STT provider: 'browser' (Web Speech API) or 'server' (OpenAI-compat) + // STT provider: 'browser' (Web Speech API), 'server' (OpenAI-compat), 'wasm' (local Whisper) sttProvider: (() => { if (typeof window !== 'undefined') { const saved = localStorage.getItem('sttProvider'); - if (saved === 'browser' || saved === 'server') return saved; + if (saved === 'browser' || saved === 'server' || saved === 'wasm') return saved; + // Electron/Chromium's Web Speech API requires Google API keys + // not available in Electron, so default to WASM local Whisper. + const electron = (window as unknown as { __OPENCHAMBER_ELECTRON__?: { runtime?: string } }).__OPENCHAMBER_ELECTRON__; + if (electron?.runtime === 'electron') return 'wasm' as const; } return 'browser' as const; })(), @@ -783,6 +790,13 @@ export const useConfigStore = create()( } return 'deepdml/faster-whisper-large-v3-turbo-ct2'; })(), + wasmSttModel: (() => { + if (typeof window !== 'undefined') { + const saved = localStorage.getItem('wasmSttModel'); + if (saved) return saved; + } + return 'Xenova/whisper-base.en'; + })(), sttLanguage: (() => { if (typeof window !== 'undefined') { const saved = localStorage.getItem('sttLanguage'); @@ -1886,7 +1900,7 @@ export const useConfigStore = create()( } }, - setSttProvider: (provider: 'browser' | 'server') => { + setSttProvider: (provider: 'browser' | 'server' | 'wasm') => { set({ sttProvider: provider }); if (typeof window !== 'undefined') { localStorage.setItem('sttProvider', provider); @@ -1910,6 +1924,14 @@ export const useConfigStore = create()( updateDesktopSettings({ sttModel: model }).catch(() => {}); }, + setWasmSttModel: (model: string) => { + set({ wasmSttModel: model }); + if (typeof window !== 'undefined') { + localStorage.setItem('wasmSttModel', model); + } + updateDesktopSettings({ wasmSttModel: model }).catch(() => {}); + }, + setSttLanguage: (lang: string) => { set({ sttLanguage: lang }); if (typeof window !== 'undefined') { diff --git a/packages/web/server/lib/opencode/settings-helpers.js b/packages/web/server/lib/opencode/settings-helpers.js index 158cc955..3bd6c31e 100644 --- a/packages/web/server/lib/opencode/settings-helpers.js +++ b/packages/web/server/lib/opencode/settings-helpers.js @@ -604,7 +604,7 @@ export const createSettingsHelpers = (dependencies) => { if (typeof candidate.sttProvider === 'string') { const provider = candidate.sttProvider.trim(); - if (provider === 'browser' || provider === 'server') { + if (provider === 'browser' || provider === 'server' || provider === 'wasm') { result.sttProvider = provider; } } @@ -620,6 +620,12 @@ export const createSettingsHelpers = (dependencies) => { result.sttModel = trimmed; } } + if (typeof candidate.wasmSttModel === 'string') { + const trimmed = candidate.wasmSttModel.trim(); + if (trimmed.length <= 256) { + result.wasmSttModel = trimmed; + } + } if (typeof candidate.sttLanguage === 'string') { const trimmed = candidate.sttLanguage.trim(); if (trimmed.length <= STT_LANGUAGE_MAX_LENGTH) {