From 391f9383345d065717228966fba0e7d9b50cd735 Mon Sep 17 00:00:00 2001 From: Bohdan Triapitsyn Date: Sun, 30 Aug 2026 02:24:11 +0300 Subject: [PATCH] feat(voice): match local and macOS voices to the language of the text Text-to-speech picked one voice regardless of what language a reply was in. A dependency-free language detector (script, marker letters, function words) now decides the language of the whole message once; with the new "Match the voice to the language of the text" setting the local provider switches to a catalog model for that language (Kokoro zh/en and Piper models for 12 languages, downloaded on first use like the existing model) and macOS say switches to an installed voice whose locale matches. The local voice picker lists voices of every installed model, and the settings show which language models are on disk. The Ukrainian Piper medium build is a character-level model that sherpa-onnx turns into noise, so the espeak-based Lada build is used instead. Claude-Session: https://claude.ai/code/session_017TK5JAYDfT3Fotc23UEg98 --- .../sections/openchamber/VoiceSettings.tsx | 221 ++++++++++++------ packages/ui/src/hooks/useLocalTTS.ts | 16 +- packages/ui/src/hooks/useMessageTTS.ts | 7 + packages/ui/src/hooks/useSayTTS.ts | 3 + .../ui/src/lib/i18n/messages/de.settings.ts | 5 +- .../ui/src/lib/i18n/messages/en.settings.ts | 5 +- .../ui/src/lib/i18n/messages/es.settings.ts | 5 +- .../ui/src/lib/i18n/messages/fr.settings.ts | 5 +- .../ui/src/lib/i18n/messages/ja.settings.ts | 5 +- .../ui/src/lib/i18n/messages/ko.settings.ts | 5 +- .../ui/src/lib/i18n/messages/pl.settings.ts | 5 +- .../src/lib/i18n/messages/pt-BR.settings.ts | 5 +- .../ui/src/lib/i18n/messages/tr.settings.ts | 5 +- .../ui/src/lib/i18n/messages/uk.settings.ts | 5 +- .../src/lib/i18n/messages/zh-CN.settings.ts | 5 +- .../src/lib/i18n/messages/zh-TW.settings.ts | 5 +- packages/ui/src/stores/useConfigStore.ts | 35 +++ .../web/server/lib/dictation/DOCUMENTATION.md | 25 +- .../lib/dictation/local/model-catalog.js | 220 +++++++++++++++++ .../lib/dictation/local/model-catalog.test.js | 42 ++++ .../server/lib/dictation/local/sherpa-tts.js | 61 +++-- .../lib/dictation/local/worker-process.js | 2 + packages/web/server/lib/dictation/runtime.js | 4 + packages/web/server/lib/dictation/service.js | 31 ++- packages/web/server/lib/tts/DOCUMENTATION.md | 1 + .../web/server/lib/tts/language-detect.js | 210 +++++++++++++++++ .../server/lib/tts/language-detect.test.js | 76 ++++++ packages/web/server/lib/tts/routes.js | 24 +- packages/web/server/lib/tts/routes.test.js | 26 +++ 29 files changed, 949 insertions(+), 115 deletions(-) create mode 100644 packages/web/server/lib/dictation/local/model-catalog.test.js create mode 100644 packages/web/server/lib/tts/language-detect.js create mode 100644 packages/web/server/lib/tts/language-detect.test.js diff --git a/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx b/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx index 8d3a1ac3..9abfeffe 100644 --- a/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx +++ b/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx @@ -71,6 +71,7 @@ const LOCAL_STT_MODELS = [ interface DictationModelState { id: string; + description?: string; installed: boolean; downloading: boolean; downloadProgress: number | null; @@ -288,10 +289,32 @@ const KOKORO_VOICE_OPTIONS = [ const LOCAL_TTS_MODEL_ID = 'kokoro-en-v0_19'; -const LocalTtsModelStatus = () => { - const { t } = useI18n(); - const [model, setModel] = useState(null); - const [requesting, setRequesting] = useState(false); +const KOKORO_MULTI_LANG_MODEL_ID = 'kokoro-multi-lang-v1_1'; +// A few named speakers out of the 103 in the Chinese/English Kokoro build. +const KOKORO_MULTI_LANG_VOICE_OPTIONS = [ + { id: 0, label: 'Maple (af)' }, + { id: 1, label: 'Sol (af)' }, + { id: 2, label: 'Vale (bf)' }, + { id: 3, label: 'Xiaoxiao (zf)' }, + { id: 58, label: 'Yunxi (zm)' }, +]; + +interface LocalTtsVoiceOption { + modelId: string; + speakerId: number; + label: string; +} + +const localTtsVoiceKey = (modelId: string, speakerId: number): string => `${modelId}:${speakerId}`; + +/** + * Local TTS models as the server reports them, plus the actions Settings + * offers on them. Shared by the model list and the voice picker so both see + * the same install state. + */ +const useLocalTtsModels = () => { + const [models, setModels] = useState([]); + const [requestingId, setRequestingId] = useState(null); const refresh = useCallback(async () => { try { @@ -300,11 +323,8 @@ const LocalTtsModelStatus = () => { return; } const data = await response.json(); - const entry = Array.isArray(data?.ttsModels) - ? data.ttsModels.find((m: DictationModelState) => m.id === LOCAL_TTS_MODEL_ID) - : null; - if (entry) { - setModel(entry); + if (Array.isArray(data?.ttsModels)) { + setModels(data.ttsModels); } } catch { // Display-only status; keep the previous state on fetch failure. @@ -315,81 +335,118 @@ const LocalTtsModelStatus = () => { void refresh(); }, [refresh]); + const anyDownloading = models.some((model) => model.downloading); useEffect(() => { - if (!model?.downloading) { + if (!anyDownloading) { return; } const interval = setInterval(() => { void refresh(); }, 2000); return () => clearInterval(interval); - }, [model?.downloading, refresh]); + }, [anyDownloading, refresh]); - const request = async (method: 'POST' | 'DELETE') => { - setRequesting(true); + const request = useCallback(async (modelId: string, method: 'POST' | 'DELETE') => { + setRequestingId(modelId); try { const path = method === 'POST' - ? `/api/dictation/models/${LOCAL_TTS_MODEL_ID}/download` - : `/api/dictation/models/${LOCAL_TTS_MODEL_ID}`; + ? `/api/dictation/models/${modelId}/download` + : `/api/dictation/models/${modelId}`; await runtimeFetch(path, { method }); await refresh(); } catch { // Status refresh reports errors. } finally { - setRequesting(false); + setRequestingId(null); } - }; + }, [refresh]); - if (!model) { + return { models, requestingId, request, refresh }; +}; + +// Voices the picker offers: Kokoro speakers for the Kokoro models, one voice +// per installed Piper model. Only installed models (plus the default) appear, +// so a language model the server fetched on its own becomes selectable once +// it is on disk. +const buildLocalTtsVoiceOptions = (models: DictationModelState[]): LocalTtsVoiceOption[] => { + const options: LocalTtsVoiceOption[] = KOKORO_VOICE_OPTIONS.map((voice) => ({ + modelId: LOCAL_TTS_MODEL_ID, + speakerId: voice.id, + label: voice.label, + })); + for (const model of models) { + if (model.id === LOCAL_TTS_MODEL_ID || !model.installed) continue; + if (model.id === KOKORO_MULTI_LANG_MODEL_ID) { + for (const voice of KOKORO_MULTI_LANG_VOICE_OPTIONS) { + options.push({ modelId: model.id, speakerId: voice.id, label: `${voice.label} · Kokoro zh/en` }); + } + continue; + } + options.push({ modelId: model.id, speakerId: 0, label: model.description ?? model.id }); + } + return options; +}; + +const LocalTtsModelStatus = ({ models, requestingId, request }: ReturnType) => { + const { t } = useI18n(); + + // The default English model is always listed; language models the server + // fetched on its own appear once they are installed or downloading, so + // the list shows what is on disk rather than the whole catalog. + const visible = models.filter((model) => model.id === LOCAL_TTS_MODEL_ID || model.installed || model.downloading); + if (visible.length === 0) { return null; } return ( -
- Kokoro - 305 MB - {model.installed ? ( - <> - - - - ) : model.downloading ? ( - - - - {typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''} - - - ) : ( - - )} - {model.downloadError ? ( - {model.downloadError} - ) : null} +
+ {visible.map((model) => ( +
+ {model.description ?? model.id} + {model.installed ? ( + <> + + + + ) : model.downloading ? ( + + + + {typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''} + + + ) : ( + + )} + {model.downloadError ? ( + {model.downloadError} + ) : null} +
+ ))}
); }; @@ -424,6 +481,12 @@ export const VoiceSettings: React.FC = () => { const sayVoice = useConfigStore((state) => state.sayVoice); const setSayVoice = useConfigStore((state) => state.setSayVoice); const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId); + const localTtsModelId = useConfigStore((state) => state.localTtsModelId); + const setLocalTtsModelId = useConfigStore((state) => state.setLocalTtsModelId); + const localTtsModels = useLocalTtsModels(); + const localTtsVoiceOptions = useMemo(() => buildLocalTtsVoiceOptions(localTtsModels.models), [localTtsModels.models]); + const ttsFollowTextLanguage = useConfigStore((state) => state.ttsFollowTextLanguage); + const setTtsFollowTextLanguage = useConfigStore((state) => state.setTtsFollowTextLanguage); const setLocalTtsVoiceId = useConfigStore((state) => state.setLocalTtsVoiceId); const { speak: speakLocalTts, stop: stopLocalTts, isPlaying: isLocalTtsPlaying, error: localTtsError } = useLocalTTS(); @@ -432,13 +495,14 @@ export const VoiceSettings: React.FC = () => { stopLocalTts(); return; } - const voiceLabel = KOKORO_VOICE_OPTIONS.find((v) => v.id === localTtsVoiceId)?.label + const voiceLabel = localTtsVoiceOptions.find((v) => v.modelId === localTtsModelId && v.speakerId === localTtsVoiceId)?.label ?? String(localTtsVoiceId); void speakLocalTts(t('settings.voice.page.preview.voiceLine', { voiceName: voiceLabel }), { + model: localTtsModelId, speakerId: localTtsVoiceId, speed: useConfigStore.getState().speechRate, }); - }, [isLocalTtsPlaying, localTtsVoiceId, speakLocalTts, stopLocalTts, t]); + }, [isLocalTtsPlaying, localTtsModelId, localTtsVoiceId, localTtsVoiceOptions, speakLocalTts, stopLocalTts, t]); const browserVoice = useConfigStore((state) => state.browserVoice); const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice); const openaiVoice = useConfigStore((state) => state.openaiVoice); @@ -959,24 +1023,39 @@ export const VoiceSettings: React.FC = () => { )} {/* Local (Kokoro) TTS model status */} - {voiceProvider === 'local' && } + {voiceProvider === 'local' && } + + {(voiceProvider === 'local' || voiceProvider === 'say') && ( + + )} {/* Voice Selection */} {voiceProvider === 'local' && ( <> diff --git a/packages/ui/src/hooks/useLocalTTS.ts b/packages/ui/src/hooks/useLocalTTS.ts index 307356a2..0efc1a46 100644 --- a/packages/ui/src/hooks/useLocalTTS.ts +++ b/packages/ui/src/hooks/useLocalTTS.ts @@ -14,10 +14,18 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { runtimeFetch } from '@/lib/runtime-fetch'; export interface LocalTTSSpeakOptions { - /** Kokoro speaker id (0-10) */ + /** Catalog id of the local model to use; defaults to the server's default model. */ + model?: string; + /** Speaker id within the model (Kokoro voices; Piper models have one) */ speakerId?: number; /** Playback speed multiplier (1.0 = normal) */ speed?: number; + /** + * `'auto'`: the server picks a model and voice for the text's language. + * The language is judged on the whole message, not on each chunk sent for + * synthesis, so a short chunk cannot flip the voice mid-reply. + */ + language?: 'auto'; onStart?: () => void; onEnd?: () => void; onError?: (error: string) => void; @@ -35,6 +43,8 @@ export interface UseLocalTTSReturn { /** Target chunk size: big enough to amortize requests, small enough for low latency. */ const MIN_CHUNK_CHARS = 60; const MAX_CHUNK_CHARS = 400; +// Enough of the message for language detection to see whole sentences. +const LANGUAGE_SAMPLE_CHARS = 2000; /** * Split text into sentence-aligned chunks for pipelined synthesis. @@ -170,6 +180,7 @@ export function useLocalTTS(): UseLocalTTSReturn { const session: PlaybackSession = { cancelled: false, abort: new AbortController() }; sessionRef.current = session; + const languageSample = options?.language === 'auto' ? text.slice(0, LANGUAGE_SAMPLE_CHARS) : undefined; const fetchChunk = async (chunk: string): Promise => { const response = await runtimeFetch('/api/dictation/tts/speak', { @@ -177,8 +188,11 @@ export function useLocalTTS(): UseLocalTTSReturn { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: chunk, + model: options?.model, ...(typeof options?.speakerId === 'number' ? { speakerId: options.speakerId } : {}), ...(typeof options?.speed === 'number' ? { speed: options.speed } : {}), + language: options?.language, + languageSample, }), signal: session.abort.signal, }); diff --git a/packages/ui/src/hooks/useMessageTTS.ts b/packages/ui/src/hooks/useMessageTTS.ts index 88e10615..4950e320 100644 --- a/packages/ui/src/hooks/useMessageTTS.ts +++ b/packages/ui/src/hooks/useMessageTTS.ts @@ -61,6 +61,8 @@ export function useMessageTTS(): UseMessageTTSReturn { const speechVolume = useConfigStore((state) => state.speechVolume); const sayVoice = useConfigStore((state) => state.sayVoice); const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId); + const localTtsModelId = useConfigStore((state) => state.localTtsModelId); + const ttsFollowTextLanguage = useConfigStore((state) => state.ttsFollowTextLanguage); const browserVoice = useConfigStore((state) => state.browserVoice); const openaiVoice = useConfigStore((state) => state.openaiVoice); const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice); @@ -135,8 +137,10 @@ export function useMessageTTS(): UseMessageTTSReturn { }); } else if (voiceProvider === 'local') { await speakLocalTTS(sanitizedText, { + model: localTtsModelId, speakerId: localTtsVoiceId, speed: speechRate, + language: ttsFollowTextLanguage ? 'auto' : undefined, onEnd: () => setIsPlaying(false), onError: () => setIsPlaying(false), }); @@ -145,6 +149,7 @@ export function useMessageTTS(): UseMessageTTSReturn { await speakSayTTS(sanitizedText, { voice: sayVoice, rate: wordsPerMinute, + language: ttsFollowTextLanguage ? 'auto' : undefined, onEnd: () => setIsPlaying(false), onError: () => setIsPlaying(false), }); @@ -187,6 +192,8 @@ export function useMessageTTS(): UseMessageTTSReturn { speakSayTTS, speakLocalTTS, localTtsVoiceId, + localTtsModelId, + ttsFollowTextLanguage, stop, ]); diff --git a/packages/ui/src/hooks/useSayTTS.ts b/packages/ui/src/hooks/useSayTTS.ts index c00353b1..f9108e6d 100644 --- a/packages/ui/src/hooks/useSayTTS.ts +++ b/packages/ui/src/hooks/useSayTTS.ts @@ -105,6 +105,8 @@ interface SpeakOptions { voice?: string; /** Speech rate in words per minute (defaults to 200) */ rate?: number; + /** `'auto'`: the server switches to a voice that speaks the text's language. */ + language?: 'auto'; /** Callback when playback starts */ onStart?: () => void; /** Callback when playback ends */ @@ -229,6 +231,7 @@ export function useSayTTS(options: UseSayTTSOptions = {}): UseSayTTSReturn { text: text.trim(), voice: options?.voice || 'Samantha', rate: options?.rate || 200, + language: options?.language, }), signal: abortControllerRef.current.signal, }); diff --git a/packages/ui/src/lib/i18n/messages/de.settings.ts b/packages/ui/src/lib/i18n/messages/de.settings.ts index 75d596f9..86dc0985 100644 --- a/packages/ui/src/lib/i18n/messages/de.settings.ts +++ b/packages/ui/src/lib/i18n/messages/de.settings.ts @@ -1795,7 +1795,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Server', 'settings.voice.page.provider.local': 'Lokal', 'settings.voice.page.tooltip.sttLocal': 'On-device Transkription auf dem OpenChamber-Server. Modelle werden automatisch heruntergeladen; kein API-Schlüssel erforderlich.', - 'settings.voice.page.tooltip.localTts': 'On-device Synthese auf dem OpenChamber-Server (Kokoro, Englisch). Das Modell wird automatisch heruntergeladen; kein API-Schlüssel erforderlich.', + 'settings.voice.page.tooltip.localTts': 'On-Device-Synthese auf dem OpenChamber-Server (Kokoro für Englisch; Modelle für andere Sprachen werden beim ersten Einsatz geladen). Kein API-Schlüssel nötig.', + 'settings.voice.page.field.followTextLanguage': 'Stimme an die Sprache des Textes anpassen', + 'settings.voice.page.field.followTextLanguageAria': 'Stimme an die Sprache des Textes anpassen', + 'settings.voice.page.field.followTextLanguageInfo': 'Ist eine Antwort in einer anderen Sprache, wird eine Stimme für diese Sprache verwendet: eine passende macOS-Stimme oder ein lokales Modell, das beim ersten Einsatz geladen wird.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (Englisch)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 europäische Sprachen)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (mehrsprachig)', diff --git a/packages/ui/src/lib/i18n/messages/en.settings.ts b/packages/ui/src/lib/i18n/messages/en.settings.ts index 95620f19..e0c6468f 100644 --- a/packages/ui/src/lib/i18n/messages/en.settings.ts +++ b/packages/ui/src/lib/i18n/messages/en.settings.ts @@ -1862,7 +1862,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Server', 'settings.voice.page.provider.local': 'Local', 'settings.voice.page.tooltip.sttLocal': 'On-device transcription on the OpenChamber server. Models download automatically; no API key needed.', - 'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro, English). The model downloads automatically; no API key needed.', + 'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro for English; models for other languages download on first use). No API key needed.', + 'settings.voice.page.field.followTextLanguage': 'Match the voice to the language of the text', + 'settings.voice.page.field.followTextLanguageAria': 'Match the voice to the language of the text', + 'settings.voice.page.field.followTextLanguageInfo': 'When a reply is in another language, a voice for that language is used: a matching macOS voice, or a local model that downloads on first use.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (English)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 European languages)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingual)', diff --git a/packages/ui/src/lib/i18n/messages/es.settings.ts b/packages/ui/src/lib/i18n/messages/es.settings.ts index fa40f214..b72f9e26 100644 --- a/packages/ui/src/lib/i18n/messages/es.settings.ts +++ b/packages/ui/src/lib/i18n/messages/es.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { "settings.voice.page.provider.server": "Servidor", "settings.voice.page.provider.local": "Local", "settings.voice.page.tooltip.sttLocal": "Transcripción local en el servidor de OpenChamber. Los modelos se descargan automáticamente; no se necesita clave de API.", - "settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro, inglés). El modelo se descarga automáticamente; no se necesita clave de API.", + "settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro para inglés; los modelos de otros idiomas se descargan en el primer uso). No requiere clave de API.", + "settings.voice.page.field.followTextLanguage": "Ajustar la voz al idioma del texto", + "settings.voice.page.field.followTextLanguageAria": "Ajustar la voz al idioma del texto", + "settings.voice.page.field.followTextLanguageInfo": "Si una respuesta está en otro idioma, se usa una voz para ese idioma: una voz de macOS adecuada o un modelo local que se descarga en el primer uso.", "settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglés)", "settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeos)", "settings.voice.page.stt.model.whisperBase": "Whisper base (multilingüe)", diff --git a/packages/ui/src/lib/i18n/messages/fr.settings.ts b/packages/ui/src/lib/i18n/messages/fr.settings.ts index 432d536d..ae7d5d4e 100644 --- a/packages/ui/src/lib/i18n/messages/fr.settings.ts +++ b/packages/ui/src/lib/i18n/messages/fr.settings.ts @@ -1757,7 +1757,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Serveur', 'settings.voice.page.provider.local': 'Local', 'settings.voice.page.tooltip.sttLocal': 'Transcription locale sur le serveur OpenChamber. Les modèles se téléchargent automatiquement ; aucune clé d\'API requise.', - 'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro, anglais). Le modèle se télécharge automatiquement ; aucune clé d’API requise.', + 'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro pour l’anglais ; les modèles des autres langues sont téléchargés à la première utilisation). Aucune clé API requise.', + 'settings.voice.page.field.followTextLanguage': 'Adapter la voix à la langue du texte', + 'settings.voice.page.field.followTextLanguageAria': 'Adapter la voix à la langue du texte', + 'settings.voice.page.field.followTextLanguageInfo': 'Si une réponse est dans une autre langue, une voix pour cette langue est utilisée : une voix macOS adaptée ou un modèle local téléchargé à la première utilisation.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (anglais)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 langues européennes)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingue)', diff --git a/packages/ui/src/lib/i18n/messages/ja.settings.ts b/packages/ui/src/lib/i18n/messages/ja.settings.ts index 87b35b08..a68f0fc7 100644 --- a/packages/ui/src/lib/i18n/messages/ja.settings.ts +++ b/packages/ui/src/lib/i18n/messages/ja.settings.ts @@ -1872,7 +1872,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'サーバー', 'settings.voice.page.provider.local': 'ローカル', 'settings.voice.page.tooltip.sttLocal': 'OpenChamber サーバー上でローカルに文字起こしします。モデルは自動でダウンロードされ、API キーは不要です。', - 'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します（Kokoro、英語）。モデルは自動でダウンロードされ、API キーは不要です。', + 'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します（英語は Kokoro、他の言語のモデルは初回使用時にダウンロード）。API キーは不要です。', + 'settings.voice.page.field.followTextLanguage': 'テキストの言語に合わせて音声を選ぶ', + 'settings.voice.page.field.followTextLanguageAria': 'テキストの言語に合わせて音声を選ぶ', + 'settings.voice.page.field.followTextLanguageInfo': '返答が別の言語の場合、その言語の音声を使います。対応する macOS の音声、または初回使用時にダウンロードされるローカルモデルです。', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2（英語）', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3（ヨーロッパ25言語）', 'settings.voice.page.stt.model.whisperBase': 'Whisper base（多言語）', diff --git a/packages/ui/src/lib/i18n/messages/ko.settings.ts b/packages/ui/src/lib/i18n/messages/ko.settings.ts index 08ac89c7..4fdb93af 100644 --- a/packages/ui/src/lib/i18n/messages/ko.settings.ts +++ b/packages/ui/src/lib/i18n/messages/ko.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': '서버', 'settings.voice.page.provider.local': '로컬', 'settings.voice.page.tooltip.sttLocal': 'OpenChamber 서버에서 로컬로 변환합니다. 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.', - 'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(Kokoro, 영어). 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.', + 'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(영어는 Kokoro, 다른 언어 모델은 처음 사용할 때 다운로드). API 키가 필요 없습니다.', + 'settings.voice.page.field.followTextLanguage': '텍스트 언어에 맞는 음성 사용', + 'settings.voice.page.field.followTextLanguageAria': '텍스트 언어에 맞는 음성 사용', + 'settings.voice.page.field.followTextLanguageInfo': '응답이 다른 언어이면 해당 언어의 음성을 사용합니다. 일치하는 macOS 음성 또는 처음 사용할 때 다운로드되는 로컬 모델입니다.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (영어)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (유럽 25개 언어)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (다국어)', diff --git a/packages/ui/src/lib/i18n/messages/pl.settings.ts b/packages/ui/src/lib/i18n/messages/pl.settings.ts index ac8d5357..f8272cc1 100644 --- a/packages/ui/src/lib/i18n/messages/pl.settings.ts +++ b/packages/ui/src/lib/i18n/messages/pl.settings.ts @@ -2176,7 +2176,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Serwer', 'settings.voice.page.provider.local': 'Lokalny', 'settings.voice.page.tooltip.sttLocal': 'Transkrypcja lokalna na serwerze OpenChamber. Modele pobierają się automatycznie; klucz API nie jest potrzebny.', - 'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro, angielski). Model pobiera się automatycznie; klucz API nie jest potrzebny.', + 'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro dla angielskiego; modele innych języków pobierane przy pierwszym użyciu). Klucz API nie jest potrzebny.', + 'settings.voice.page.field.followTextLanguage': 'Dopasuj głos do języka tekstu', + 'settings.voice.page.field.followTextLanguageAria': 'Dopasuj głos do języka tekstu', + 'settings.voice.page.field.followTextLanguageInfo': 'Gdy odpowiedź jest w innym języku, używany jest głos dla tego języka: pasujący głos macOS albo lokalny model pobierany przy pierwszym użyciu.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (angielski)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 języków europejskich)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (wielojęzyczny)', diff --git a/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts b/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts index fb5fa2c1..3f2827c5 100644 --- a/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts +++ b/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { "settings.voice.page.provider.server": "Servidor", "settings.voice.page.provider.local": "Local", "settings.voice.page.tooltip.sttLocal": "Transcrição local no servidor do OpenChamber. Os modelos são baixados automaticamente; não é necessária chave de API.", - "settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro, inglês). O modelo é baixado automaticamente; não é necessária chave de API.", + "settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro para inglês; modelos de outros idiomas são baixados no primeiro uso). Não requer chave de API.", + "settings.voice.page.field.followTextLanguage": "Ajustar a voz ao idioma do texto", + "settings.voice.page.field.followTextLanguageAria": "Ajustar a voz ao idioma do texto", + "settings.voice.page.field.followTextLanguageInfo": "Se uma resposta estiver em outro idioma, uma voz desse idioma é usada: uma voz do macOS correspondente ou um modelo local baixado no primeiro uso.", "settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglês)", "settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeus)", "settings.voice.page.stt.model.whisperBase": "Whisper base (multilíngue)", diff --git a/packages/ui/src/lib/i18n/messages/tr.settings.ts b/packages/ui/src/lib/i18n/messages/tr.settings.ts index c4ce2859..ead5621c 100644 --- a/packages/ui/src/lib/i18n/messages/tr.settings.ts +++ b/packages/ui/src/lib/i18n/messages/tr.settings.ts @@ -1787,7 +1787,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Sunucu', 'settings.voice.page.provider.local': 'Yerel', 'settings.voice.page.tooltip.sttLocal': 'OpenChamber sunucusunda cihaz üstü transkripsiyon. Modeller otomatik indirilir; API anahtarı gerekmez.', - 'settings.voice.page.tooltip.localTts': 'OpenChamber sunucusunda cihaz üstü sentez (Kokoro, İngilizce). Model otomatik indirilir; API anahtarı gerekmez.', + 'settings.voice.page.tooltip.localTts': 'OpenChamber sunucusunda yerel sentez (İngilizce için Kokoro; diğer dillerin modelleri ilk kullanımda indirilir). API anahtarı gerekmez.', + 'settings.voice.page.field.followTextLanguage': 'Sesi metnin diline göre seç', + 'settings.voice.page.field.followTextLanguageAria': 'Sesi metnin diline göre seç', + 'settings.voice.page.field.followTextLanguageInfo': 'Yanıt başka bir dildeyse o dil için bir ses kullanılır: uygun bir macOS sesi veya ilk kullanımda indirilen yerel bir model.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (İngilizce)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 Avrupa dili)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (çok dilli)', diff --git a/packages/ui/src/lib/i18n/messages/uk.settings.ts b/packages/ui/src/lib/i18n/messages/uk.settings.ts index d65facf2..f3d88411 100644 --- a/packages/ui/src/lib/i18n/messages/uk.settings.ts +++ b/packages/ui/src/lib/i18n/messages/uk.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { "settings.voice.page.provider.server": "Сервер", "settings.voice.page.provider.local": "Локальний", "settings.voice.page.tooltip.sttLocal": "Локальна розшифровка на сервері OpenChamber. Моделі завантажуються автоматично; ключ API не потрібен.", - "settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro, англійська). Модель завантажується автоматично; ключ API не потрібен.", + "settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro для англійської; моделі для інших мов завантажуються при першому використанні). Ключ API не потрібен.", + "settings.voice.page.field.followTextLanguage": "Підбирати голос під мову тексту", + "settings.voice.page.field.followTextLanguageAria": "Підбирати голос під мову тексту", + "settings.voice.page.field.followTextLanguageInfo": "Якщо відповідь іншою мовою, використовується голос цієї мови: відповідний голос macOS або локальна модель, яка завантажується при першому використанні.", "settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (англійська)", "settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 європейських мов)", "settings.voice.page.stt.model.whisperBase": "Whisper base (мультимовна)", diff --git a/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts b/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts index c7f1f622..c0eef58a 100644 --- a/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts +++ b/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': '服务器', 'settings.voice.page.provider.local': '本地', 'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 服务器上本地转写。模型自动下载，无需 API 密钥。', - 'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音（Kokoro，英语）。模型自动下载，无需 API 密钥。', + 'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音（英语使用 Kokoro；其他语言的模型在首次使用时下载）。无需 API 密钥。', + 'settings.voice.page.field.followTextLanguage': '根据文本语言匹配语音', + 'settings.voice.page.field.followTextLanguageAria': '根据文本语言匹配语音', + 'settings.voice.page.field.followTextLanguageInfo': '当回复使用其他语言时，将使用该语言的语音：匹配的 macOS 语音，或首次使用时下载的本地模型。', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2（英语）', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3（25 种欧洲语言）', 'settings.voice.page.stt.model.whisperBase': 'Whisper base（多语言）', diff --git a/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts b/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts index e24521ae..c0639769 100644 --- a/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts +++ b/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts @@ -1746,7 +1746,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': '伺服器', 'settings.voice.page.provider.local': '本機', 'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 伺服器上本機轉寫。模型會自動下載，無需 API 金鑰。', - 'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音（Kokoro，英文）。模型會自動下載，無需 API 金鑰。', + 'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音（英文使用 Kokoro；其他語言的模型在首次使用時下載）。不需要 API 金鑰。', + 'settings.voice.page.field.followTextLanguage': '依文字語言選擇語音', + 'settings.voice.page.field.followTextLanguageAria': '依文字語言選擇語音', + 'settings.voice.page.field.followTextLanguageInfo': '當回覆使用其他語言時，會使用該語言的語音：相符的 macOS 語音，或首次使用時下載的本機模型。', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2（英文）', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3（25 種歐洲語言）', 'settings.voice.page.stt.model.whisperBase': 'Whisper base（多語言）', diff --git a/packages/ui/src/stores/useConfigStore.ts b/packages/ui/src/stores/useConfigStore.ts index b4a370ca..66f1487e 100644 --- a/packages/ui/src/stores/useConfigStore.ts +++ b/packages/ui/src/stores/useConfigStore.ts @@ -1067,6 +1067,10 @@ interface ConfigStore { sayVoice: string; browserVoice: string; localTtsVoiceId: number; + /** Local TTS model the chosen voice belongs to (catalog id). */ + localTtsModelId: string; + /** Local and macOS voices follow the language of the text being read. */ + ttsFollowTextLanguage: boolean; openaiVoice: string; openaiApiKey: string; openaiCompatibleUrl: string; @@ -1094,6 +1098,8 @@ interface ConfigStore { setSayVoice: (voice: string) => void; setBrowserVoice: (voice: string) => void; setLocalTtsVoiceId: (voiceId: number) => void; + setLocalTtsModelId: (modelId: string) => void; + setTtsFollowTextLanguage: (enabled: boolean) => void; setOpenaiVoice: (voice: string) => void; setOpenaiApiKey: (apiKey: string) => void; setOpenaiCompatibleUrl: (url: string) => void; @@ -1277,6 +1283,21 @@ export const useConfigStore = create()( } return 0; })(), + localTtsModelId: (() => { + if (typeof window !== 'undefined') { + const saved = localStorage.getItem('localTtsModelId'); + if (saved) return saved; + } + return 'kokoro-en-v0_19'; + })(), + + ttsFollowTextLanguage: (() => { + if (typeof window !== 'undefined') { + const saved = localStorage.getItem('ttsFollowTextLanguage'); + if (saved !== null) return saved === 'true'; + } + return true; + })(), // Browser voice - load from localStorage or default to empty (auto-select) browserVoice: (() => { if (typeof window !== 'undefined') { @@ -2962,6 +2983,20 @@ export const useConfigStore = create()( } }, + setLocalTtsModelId: (modelId: string) => { + set({ localTtsModelId: modelId }); + if (typeof window !== 'undefined') { + localStorage.setItem('localTtsModelId', modelId); + } + }, + + setTtsFollowTextLanguage: (enabled: boolean) => { + set({ ttsFollowTextLanguage: enabled }); + if (typeof window !== 'undefined') { + localStorage.setItem('ttsFollowTextLanguage', String(enabled)); + } + }, + setBrowserVoice: (voice: string) => { set({ browserVoice: voice }); if (typeof window !== 'undefined') { diff --git a/packages/web/server/lib/dictation/DOCUMENTATION.md b/packages/web/server/lib/dictation/DOCUMENTATION.md index e66af116..49e38aae 100644 --- a/packages/web/server/lib/dictation/DOCUMENTATION.md +++ b/packages/web/server/lib/dictation/DOCUMENTATION.md @@ -11,12 +11,25 @@ live transcript costs O(n^2) work for a result the final decode replaces. The composer shows no text while recording and inserts the full transcript on stop. -Local TTS (Kokoro via sherpa-onnx OfflineTts) runs in the same worker process -and is exposed as `POST /api/dictation/tts/speak` (JSON `{text, speakerId?, -speed?, model?}` → WAV bytes; 503 with `reasonCode` while the model is -downloading). TTS models live in the same catalog/downloader as STT models -(`local/model-catalog.js` `LOCAL_TTS_MODEL_CATALOG`) and are managed by the -same status/download/delete routes. +Local TTS (Kokoro and Piper/VITS via sherpa-onnx OfflineTts) runs in the same +worker process and is exposed as `POST /api/dictation/tts/speak` (JSON +`{text, speakerId?, speed?, model?, language?, languageSample?}` → WAV bytes; 503 with +`reasonCode` while the model is downloading). TTS models live in the same +catalog/downloader as STT models (`local/model-catalog.js` +`LOCAL_TTS_MODEL_CATALOG`) and are managed by the same status/download/delete +routes. + +Each TTS catalog entry declares the `languages` it speaks. With +`language: 'auto'` the service detects the language of `languageSample` — the +whole message the chunk belongs to, sent by the client with every chunk — or +of `text` when no sample is given +(`../tts/language-detect.js`, script plus function-word scoring, no +dependencies) and keeps the caller's model when it speaks that language; +otherwise it switches to the catalog model for the language, downloading it on +first use like any other model, and starts from that model's default speaker +(`defaultSpeakerByLanguage`) instead of the caller's speaker id. A language no +catalog model covers keeps the caller's model, so text is always spoken. The +response carries `X-Speech-Model` and `X-Speech-Language`. ## Ownership diff --git a/packages/web/server/lib/dictation/local/model-catalog.js b/packages/web/server/lib/dictation/local/model-catalog.js index 29ea524b..16af62c9 100644 --- a/packages/web/server/lib/dictation/local/model-catalog.js +++ b/packages/web/server/lib/dictation/local/model-catalog.js @@ -68,9 +68,19 @@ export const LOCAL_STT_MODEL_CATALOG = { * Local text-to-speech models (sherpa-onnx OfflineTts). Downloaded and * managed through the same pipeline as the STT models. */ +/** + * Local text-to-speech models (sherpa-onnx OfflineTts). Downloaded and + * managed through the same pipeline as the STT models. + * + * `languages` lists the languages a model speaks well; the speech service + * uses it to pick a model for the language a text is written in. Kokoro + * models carry speaker ids (`voices`); a Piper model is one voice for one + * language. `lexicon` entries are joined with commas for sherpa-onnx. + */ export const LOCAL_TTS_MODEL_CATALOG = { 'kokoro-en-v0_19': { type: 'kokoro', + languages: ['en'], archiveUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2', extractedDir: 'kokoro-en-v0_19', @@ -82,6 +92,188 @@ export const LOCAL_TTS_MODEL_CATALOG = { }, description: 'Kokoro TTS (English, natural voices)', }, + 'kokoro-multi-lang-v1_1': { + type: 'kokoro', + languages: ['zh', 'en'], + // sherpa-onnx wires this Kokoro build for Chinese and English only; + // speakers 0-2 are English, 3-102 Chinese. + defaultSpeakerByLanguage: { en: 0, zh: 3 }, + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_1.tar.bz2', + extractedDir: 'kokoro-multi-lang-v1_1', + files: { + model: 'model.onnx', + voices: 'voices.bin', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + lexiconEnglish: 'lexicon-us-en.txt', + lexiconChinese: 'lexicon-zh.txt', + }, + lexicon: ['lexiconEnglish', 'lexiconChinese'], + description: 'Kokoro TTS (Chinese and English, 103 voices)', + }, + // The larger `ukrainian_tts-medium` build is a character-level model + // (`phoneme_type: text`); sherpa-onnx phonemizes every Piper model through + // espeak-ng, which turns that one into noise. `vits-coqui-uk-mai` sounds + // better but reads Cyrillic only and drops every Latin word (file names, + // product names), which is unusable in a coding chat. Lada is an espeak + // model: small, but it reads mixed text. + 'piper-uk_UA-lada-x_low': { + type: 'vits', + languages: ['uk'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-uk_UA-lada-x_low.tar.bz2', + extractedDir: 'vits-piper-uk_UA-lada-x_low', + files: { + model: 'uk_UA-lada-x_low.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Ukrainian)', + }, + 'piper-de_DE-thorsten-medium': { + type: 'vits', + languages: ['de'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-thorsten-medium.tar.bz2', + extractedDir: 'vits-piper-de_DE-thorsten-medium', + files: { + model: 'de_DE-thorsten-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (German)', + }, + 'piper-fr_FR-siwis-medium': { + type: 'vits', + languages: ['fr'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-fr_FR-siwis-medium.tar.bz2', + extractedDir: 'vits-piper-fr_FR-siwis-medium', + files: { + model: 'fr_FR-siwis-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (French)', + }, + 'piper-es_ES-davefx-medium': { + type: 'vits', + languages: ['es'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-es_ES-davefx-medium.tar.bz2', + extractedDir: 'vits-piper-es_ES-davefx-medium', + files: { + model: 'es_ES-davefx-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Spanish)', + }, + 'piper-it_IT-paola-medium': { + type: 'vits', + languages: ['it'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-it_IT-paola-medium.tar.bz2', + extractedDir: 'vits-piper-it_IT-paola-medium', + files: { + model: 'it_IT-paola-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Italian)', + }, + 'piper-pt_BR-faber-medium': { + type: 'vits', + languages: ['pt'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-pt_BR-faber-medium.tar.bz2', + extractedDir: 'vits-piper-pt_BR-faber-medium', + files: { + model: 'pt_BR-faber-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Portuguese (Brazil))', + }, + 'piper-pl_PL-gosia-medium': { + type: 'vits', + languages: ['pl'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-pl_PL-gosia-medium.tar.bz2', + extractedDir: 'vits-piper-pl_PL-gosia-medium', + files: { + model: 'pl_PL-gosia-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Polish)', + }, + 'piper-ru_RU-irina-medium': { + type: 'vits', + languages: ['ru'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-ru_RU-irina-medium.tar.bz2', + extractedDir: 'vits-piper-ru_RU-irina-medium', + files: { + model: 'ru_RU-irina-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Russian)', + }, + 'piper-nl_NL-pim-medium': { + type: 'vits', + languages: ['nl'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-nl_NL-pim-medium.tar.bz2', + extractedDir: 'vits-piper-nl_NL-pim-medium', + files: { + model: 'nl_NL-pim-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Dutch)', + }, + 'piper-cs_CZ-jirka-medium': { + type: 'vits', + languages: ['cs'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-cs_CZ-jirka-medium.tar.bz2', + extractedDir: 'vits-piper-cs_CZ-jirka-medium', + files: { + model: 'cs_CZ-jirka-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Czech)', + }, + 'piper-tr_TR-dfki-medium': { + type: 'vits', + languages: ['tr'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-tr_TR-dfki-medium.tar.bz2', + extractedDir: 'vits-piper-tr_TR-dfki-medium', + files: { + model: 'tr_TR-dfki-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Turkish)', + }, + 'piper-sv_SE-nst-medium': { + type: 'vits', + languages: ['sv'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-sv_SE-nst-medium.tar.bz2', + extractedDir: 'vits-piper-sv_SE-nst-medium', + files: { + model: 'sv_SE-nst-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Swedish)', + }, }; export const DEFAULT_LOCAL_STT_MODEL = 'parakeet-tdt-0.6b-v2-int8'; @@ -131,6 +323,34 @@ export function getLocalSttModelSpec(modelId) { }; } +/** + * The local TTS model to use for a language, preferring the model the user + * selected when it speaks that language. Returns null when no catalog model + * covers the language, in which case callers keep the selected model. + * @param {string} language BCP-47 primary subtag (`uk`, `zh`...) + * @param {string} [preferredModelId] + * @returns {string | null} + */ +export function resolveLocalTtsModelForLanguage(language, preferredModelId) { + const speaks = (modelId) => LOCAL_TTS_MODEL_CATALOG[modelId]?.languages?.includes(language) === true; + if (preferredModelId && speaks(preferredModelId)) return preferredModelId; + const candidate = LOCAL_TTS_MODEL_IDS.find(speaks); + return candidate ?? null; +} + +/** + * The speaker id a model should use for a language when the caller's + * speaker was chosen for another language. `undefined` keeps the caller's + * speaker. + * @param {string} modelId + * @param {string} language + * @returns {number | undefined} + */ +export function getLocalTtsDefaultSpeaker(modelId, language) { + const speaker = LOCAL_TTS_MODEL_CATALOG[modelId]?.defaultSpeakerByLanguage?.[language]; + return Number.isInteger(speaker) ? speaker : undefined; +} + /** * @param {string} modelsDir * @param {string} modelId diff --git a/packages/web/server/lib/dictation/local/model-catalog.test.js b/packages/web/server/lib/dictation/local/model-catalog.test.js new file mode 100644 index 00000000..e14a66aa --- /dev/null +++ b/packages/web/server/lib/dictation/local/model-catalog.test.js @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_LOCAL_TTS_MODEL, + LOCAL_TTS_MODEL_CATALOG, + getLocalSttModelSpec, + getLocalTtsDefaultSpeaker, + resolveLocalTtsModelForLanguage, +} from './model-catalog.js'; + +describe('local TTS catalog', () => { + it('keeps the selected model when it speaks the language', () => { + expect(resolveLocalTtsModelForLanguage('en', DEFAULT_LOCAL_TTS_MODEL)).toBe(DEFAULT_LOCAL_TTS_MODEL); + expect(resolveLocalTtsModelForLanguage('zh', 'kokoro-multi-lang-v1_1')).toBe('kokoro-multi-lang-v1_1'); + }); + + it('picks a catalog model for a language the selected model lacks', () => { + expect(resolveLocalTtsModelForLanguage('uk', DEFAULT_LOCAL_TTS_MODEL)).toBe('piper-uk_UA-lada-x_low'); + expect(resolveLocalTtsModelForLanguage('zh', DEFAULT_LOCAL_TTS_MODEL)).toBe('kokoro-multi-lang-v1_1'); + }); + + it('returns null for a language no model covers', () => { + expect(resolveLocalTtsModelForLanguage('xx', DEFAULT_LOCAL_TTS_MODEL)).toBeNull(); + }); + + it('gives Chinese a Chinese speaker on the multi-language Kokoro', () => { + expect(getLocalTtsDefaultSpeaker('kokoro-multi-lang-v1_1', 'zh')).toBe(3); + expect(getLocalTtsDefaultSpeaker('kokoro-multi-lang-v1_1', 'en')).toBe(0); + expect(getLocalTtsDefaultSpeaker('piper-uk_UA-lada-x_low', 'uk')).toBeUndefined(); + }); + + it('every TTS entry declares its languages and installable files', () => { + for (const [id, spec] of Object.entries(LOCAL_TTS_MODEL_CATALOG)) { + expect(spec.languages.length, id).toBeGreaterThan(0); + expect(spec.archiveUrl, id).toMatch(/^https:\/\/github\.com\/k2-fsa\/sherpa-onnx\/releases\/download\/tts-models\//); + const resolved = getLocalSttModelSpec(id); + expect(resolved.requiredFiles, id).toContain(spec.files.model); + for (const key of spec.lexicon ?? []) { + expect(spec.files[key], `${id} lexicon ${key}`).toBeTruthy(); + } + } + }); +}); diff --git a/packages/web/server/lib/dictation/local/sherpa-tts.js b/packages/web/server/lib/dictation/local/sherpa-tts.js index f4fa7972..589bae69 100644 --- a/packages/web/server/lib/dictation/local/sherpa-tts.js +++ b/packages/web/server/lib/dictation/local/sherpa-tts.js @@ -1,5 +1,5 @@ /** - * Sherpa-onnx offline TTS (Kokoro). Runs inside the dictation worker process + * Sherpa-onnx offline TTS (Kokoro and Piper/VITS). Runs inside the dictation worker process * only — never load the native addon in the main server process. */ @@ -23,20 +23,49 @@ function float32ToPcm16le(samples) { return Buffer.from(out.buffer, out.byteOffset, out.byteLength); } +/** + * sherpa-onnx model config for one catalog entry. Kokoro carries a voices + * bank (speaker ids) and optional lexicons; a Piper/VITS model is a single + * voice with espeak-ng phonemization. + * @param {{ modelDir: string, type?: string, files: Record, lexicon?: string[] }} config + */ +function buildModelConfig(config) { + const file = (key, label) => { + const filePath = path.join(config.modelDir, config.files[key]); + assertFileExists(filePath, label); + return filePath; + }; + const modelPath = file('model', 'TTS model'); + const tokensPath = file('tokens', 'TTS tokens'); + + if (config.type === 'vits') { + // Piper models phonemize through espeak-ng (`espeakData`); character + // models (Coqui) read the text directly and carry no espeak data. + const dataDir = config.files.espeakData ? file('espeakData', 'TTS espeak-ng dataDir') : ''; + return { vits: { model: modelPath, tokens: tokensPath, ...(dataDir ? { dataDir } : {}), lengthScale: 1.0 } }; + } + + const dataDir = file('espeakData', 'TTS espeak-ng dataDir'); + const voicesPath = file('voices', 'TTS voices'); + const lexicon = (config.lexicon ?? []).map((key) => file(key, 'TTS lexicon')).join(','); + return { + kokoro: { + model: modelPath, + voices: voicesPath, + tokens: tokensPath, + dataDir, + lengthScale: 1.0, + ...(lexicon ? { lexicon } : {}), + }, + }; +} + export class SherpaTtsEngine { /** - * @param {{ modelDir: string, files: { model: string, voices: string, tokens: string, espeakData: string }, numThreads?: number }} config + * @param {{ modelDir: string, type?: string, files: Record, lexicon?: string[], numThreads?: number }} config */ constructor(config) { - const modelPath = path.join(config.modelDir, config.files.model); - const voicesPath = path.join(config.modelDir, config.files.voices); - const tokensPath = path.join(config.modelDir, config.files.tokens); - const dataDir = path.join(config.modelDir, config.files.espeakData); - - assertFileExists(modelPath, 'TTS model'); - assertFileExists(voicesPath, 'TTS voices'); - assertFileExists(tokensPath, 'TTS tokens'); - assertFileExists(dataDir, 'TTS espeak-ng dataDir'); + const model = buildModelConfig(config); const sherpa = loadSherpaOnnxNode(); if (typeof sherpa.OfflineTts !== 'function') { @@ -44,15 +73,7 @@ export class SherpaTtsEngine { } this.tts = new sherpa.OfflineTts({ - model: { - kokoro: { - model: modelPath, - voices: voicesPath, - tokens: tokensPath, - dataDir, - lengthScale: 1.0, - }, - }, + model, numThreads: config.numThreads ?? 2, provider: 'cpu', maxNumSentences: 1, diff --git a/packages/web/server/lib/dictation/local/worker-process.js b/packages/web/server/lib/dictation/local/worker-process.js index 3a5c34b8..06a7ac8b 100644 --- a/packages/web/server/lib/dictation/local/worker-process.js +++ b/packages/web/server/lib/dictation/local/worker-process.js @@ -102,7 +102,9 @@ function getTtsEngine(modelsDir, modelId) { const spec = getLocalSttModelSpec(modelId); const created = new SherpaTtsEngine({ modelDir: getLocalSttModelDir(modelsDir, modelId), + type: spec.type, files: spec.files, + lexicon: spec.lexicon, numThreads: 2, }); ttsEngines.set(key, created); diff --git a/packages/web/server/lib/dictation/runtime.js b/packages/web/server/lib/dictation/runtime.js index 8fae1fcf..59437a12 100644 --- a/packages/web/server/lib/dictation/runtime.js +++ b/packages/web/server/lib/dictation/runtime.js @@ -63,6 +63,8 @@ export function createDictationRuntime({ model: typeof req.body?.model === 'string' ? req.body.model : undefined, speakerId: Number.isInteger(req.body?.speakerId) ? req.body.speakerId : undefined, speed: typeof req.body?.speed === 'number' ? req.body.speed : undefined, + language: req.body?.language === 'auto' ? 'auto' : undefined, + languageSample: typeof req.body?.languageSample === 'string' ? req.body.languageSample.slice(0, 4000) : undefined, }); if (result.error) { res.status(503).json({ @@ -73,6 +75,8 @@ export function createDictationRuntime({ return; } res.setHeader('Content-Type', result.format || 'audio/wav'); + res.setHeader('X-Speech-Model', result.modelId); + if (result.language) res.setHeader('X-Speech-Language', result.language); res.send(result.audio); } catch (error) { res.status(500).json({ error: error?.message || 'Failed to synthesize speech' }); diff --git a/packages/web/server/lib/dictation/service.js b/packages/web/server/lib/dictation/service.js index 7dc555d2..e40bc253 100644 --- a/packages/web/server/lib/dictation/service.js +++ b/packages/web/server/lib/dictation/service.js @@ -1,3 +1,4 @@ +import { detectTextLanguage } from '../tts/language-detect.js'; /** * Dictation service: resolves STT providers, tracks local model download * state, and exposes a readiness snapshot for the status route. @@ -16,6 +17,8 @@ import { OpenAICompatibleTranscriptionSession } from './openai-compatible-sessio import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, + getLocalTtsDefaultSpeaker, + resolveLocalTtsModelForLanguage, LOCAL_STT_MODEL_CATALOG, LOCAL_STT_MODEL_IDS, LOCAL_TTS_MODEL_CATALOG, @@ -220,10 +223,30 @@ export function createDictationService({ modelsDir }) { /** * Synthesize speech with the local TTS model. Returns WAV bytes, or a * readiness error while the model is missing/downloading. - * @param {{ text: string, model?: string, speakerId?: number, speed?: number }} options + * + * With `language: 'auto'` the text's language decides the model: the + * caller's model when it speaks that language, otherwise the catalog + * model for it (downloaded on first use, reported as in-progress until it + * lands). The caller's speaker id is kept only on the caller's model; a + * substitute model starts from its own default speaker for the language. + * A language no catalog model covers keeps the caller's model, so text is + * never silently dropped. + * `languageSample` is the whole message the chunk belongs to (or a prefix + * of it): the language is judged on that, never on a short chunk alone. + * @param {{ text: string, model?: string, speakerId?: number, speed?: number, language?: string, languageSample?: string }} options */ - const synthesizeSpeech = async ({ text, model, speakerId, speed }) => { - const modelId = isLocalTtsModelId(model) ? model : DEFAULT_LOCAL_TTS_MODEL; + const synthesizeSpeech = async ({ text, model, speakerId, speed, language, languageSample }) => { + const requestedModelId = isLocalTtsModelId(model) ? model : DEFAULT_LOCAL_TTS_MODEL; + let modelId = requestedModelId; + let resolvedLanguage = null; + if (language === 'auto') { + resolvedLanguage = detectTextLanguage(languageSample || text).language; + const forLanguage = resolveLocalTtsModelForLanguage(resolvedLanguage, requestedModelId); + if (forLanguage && forLanguage !== requestedModelId) { + modelId = forLanguage; + speakerId = getLocalTtsDefaultSpeaker(modelId, resolvedLanguage); + } + } const installed = await isLocalSttModelInstalled(modelsDir, modelId); if (!installed) { const state = downloadStates.get(modelId); @@ -251,7 +274,7 @@ export function createDictationService({ modelsDir }) { speakerId, speed, }); - return { audio: result.audio, format: result.format }; + return { audio: result.audio, format: result.format, modelId, language: resolvedLanguage }; }; /** diff --git a/packages/web/server/lib/tts/DOCUMENTATION.md b/packages/web/server/lib/tts/DOCUMENTATION.md index 81a46760..2e0c4352 100644 --- a/packages/web/server/lib/tts/DOCUMENTATION.md +++ b/packages/web/server/lib/tts/DOCUMENTATION.md @@ -11,6 +11,7 @@ This module provides server-side Text-to-Speech services using OpenAI's TTS API. - `packages/web/server/lib/text/summarization.js`: Shared text summarization stub and sanitization utilities. It performs no external Zen calls. - `packages/web/server/lib/tts/stt.js`: STT proxy for OpenAI-compatible transcription endpoints. - `packages/web/server/lib/tts/base-url.js`: shared base URL validation and normalization for custom OpenAI-compatible endpoints. +- `packages/web/server/lib/tts/language-detect.js`: dependency-free language detection for voice selection (`detectTextLanguage`, `pickVoiceForLanguage`, `languageOfLocale`). Used by the macOS `say` route (`language: 'auto'` switches to an installed voice whose locale matches the text; the response carries `X-Speech-Voice` and `X-Speech-Language`) and by the dictation module's local TTS model choice. ## Public exports diff --git a/packages/web/server/lib/tts/language-detect.js b/packages/web/server/lib/tts/language-detect.js new file mode 100644 index 00000000..d87968e6 --- /dev/null +++ b/packages/web/server/lib/tts/language-detect.js @@ -0,0 +1,210 @@ +/** + * Language detection for text-to-speech voice selection. + * + * Picks the language a piece of chat text is written in so a TTS provider + * can choose a matching voice or model. Deliberately small and dependency + * free: the writing system decides most cases outright, and Latin-script + * languages are told apart by function words and characteristic letters. + * The answer is a best effort for voice selection, not a linguistic claim — + * an unknown language falls back to English rather than failing. + */ + +const SCRIPT_RANGES = [ + ['hangul', /[가-힯ᄀ-ᇿ㄰-㆏]/g], + ['kana', /[぀-ヿ]/g], + ['han', /[一-鿿㐀-䶿]/g], + ['cyrillic', /[Ѐ-ӿ]/g], + ['greek', /[Ͱ-Ͽ]/g], + ['arabic', /[؀-ۿ]/g], + ['hebrew', /[֐-׿]/g], + ['thai', /[฀-๿]/g], + ['devanagari', /[ऀ-ॿ]/g], + ['latin', /[A-Za-zÀ-ɏ]/g], +]; + +const SCRIPT_LANGUAGE = { + hangul: 'ko', + greek: 'el', + arabic: 'ar', + hebrew: 'he', + thai: 'th', + devanagari: 'hi', +}; + +// Letters that only (or overwhelmingly) occur in one language of a script. +const LATIN_MARKERS = { + pl: /[łęąńśźż]/i, + cs: /[řěůťďň]/i, + tr: /[ğışİ]/, + pt: /[ãõ]/i, + es: /[ñ¿¡]/, + de: /[ß]/, + fr: /[œ]/i, + sv: /[å]/i, +}; + +// Frequent function words per language. Scored by whole-word hits; every +// list has the same length so scores stay comparable. +const STOPWORDS = { + en: ['the', 'and', 'is', 'to', 'of', 'that', 'you', 'with', 'for', 'this', 'are', 'it', 'not', 'have', 'can', 'will', 'your', 'from', 'which', 'when'], + de: ['und', 'der', 'die', 'das', 'ist', 'nicht', 'mit', 'ein', 'eine', 'auch', 'sich', 'auf', 'für', 'wird', 'werden', 'oder', 'aber', 'wenn', 'sind', 'kann'], + fr: ['le', 'la', 'les', 'et', 'est', 'une', 'des', 'pour', 'que', 'qui', 'dans', 'pas', 'vous', 'sur', 'avec', 'sont', 'nous', 'cette', 'mais', 'plus'], + es: ['el', 'la', 'los', 'las', 'que', 'es', 'una', 'por', 'para', 'con', 'del', 'como', 'pero', 'más', 'este', 'esta', 'son', 'tiene', 'puede', 'también'], + it: ['il', 'la', 'che', 'di', 'è', 'una', 'per', 'non', 'con', 'del', 'della', 'come', 'sono', 'anche', 'questo', 'questa', 'gli', 'nel', 'più', 'essere'], + pt: ['o', 'a', 'os', 'as', 'que', 'é', 'uma', 'para', 'com', 'não', 'do', 'da', 'como', 'mas', 'também', 'este', 'esta', 'são', 'você', 'pode'], + pl: ['i', 'nie', 'jest', 'się', 'na', 'to', 'że', 'jak', 'ale', 'dla', 'oraz', 'przez', 'czy', 'tym', 'jego', 'można', 'jeśli', 'tego', 'które', 'także'], + nl: ['de', 'het', 'een', 'en', 'van', 'is', 'niet', 'dat', 'met', 'voor', 'ook', 'zijn', 'maar', 'als', 'wordt', 'deze', 'kan', 'naar', 'bij', 'dan'], + cs: ['a', 'je', 'se', 'na', 'to', 'že', 'jak', 'ale', 'pro', 'nebo', 'jsou', 'může', 'také', 'tento', 'když', 'jeho', 'které', 'být', 'aby', 'ještě'], + tr: ['ve', 'bir', 'bu', 'için', 'ile', 'de', 'da', 'ama', 'gibi', 'daha', 'var', 'olarak', 'çok', 'ne', 'her', 'kadar', 'sonra', 'değil', 'olan', 'ise'], + sv: ['och', 'att', 'det', 'är', 'en', 'som', 'för', 'inte', 'med', 'till', 'den', 'kan', 'har', 'ett', 'men', 'också', 'eller', 'från', 'när', 'vara'], + uk: ['і', 'та', 'що', 'це', 'не', 'як', 'для', 'він', 'вона', 'але', 'або', 'також', 'тільки', 'вже', 'якщо', 'його', 'цей', 'ця', 'бути', 'коли'], + ru: ['и', 'что', 'это', 'не', 'как', 'для', 'он', 'она', 'но', 'или', 'также', 'только', 'уже', 'если', 'его', 'этот', 'эта', 'быть', 'когда', 'чтобы'], +}; + +const LATIN_LANGUAGES = ['en', 'de', 'fr', 'es', 'it', 'pt', 'pl', 'nl', 'cs', 'tr', 'sv']; +const CYRILLIC_LANGUAGES = ['uk', 'ru']; + +const countMatches = (text, pattern) => { + const matches = text.match(pattern); + return matches ? matches.length : 0; +}; + +const scoreStopwords = (words, languages) => { + const scores = {}; + for (const language of languages) { + const list = new Set(STOPWORDS[language]); + let hits = 0; + for (const word of words) { + if (list.has(word)) hits += 1; + } + scores[language] = hits; + } + return scores; +}; + +const bestOf = (scores, fallback) => { + let best = fallback; + let bestScore = 0; + for (const [language, score] of Object.entries(scores)) { + if (score > bestScore) { + best = language; + bestScore = score; + } + } + return best; +}; + +const pickByMarkers = (text, markers) => { + for (const [language, pattern] of Object.entries(markers)) { + if (pattern.test(text)) return language; + } + return null; +}; + +/** + * @param {string} text + * @returns {{ language: string, script: string }} BCP-47 primary language subtag and the dominant script. + */ +export function detectTextLanguage(text) { + const source = typeof text === 'string' ? text : ''; + const counts = SCRIPT_RANGES.map(([script, pattern]) => [script, countMatches(source, pattern)]); + const letters = counts.reduce((sum, [, count]) => sum + count, 0); + if (letters === 0) return { language: 'en', script: 'latin' }; + + // Kana settles Japanese even when Han dominates the character count. + const kana = counts.find(([script]) => script === 'kana')?.[1] ?? 0; + const han = counts.find(([script]) => script === 'han')?.[1] ?? 0; + if (kana > 0 && kana + han >= letters * 0.3) return { language: 'ja', script: 'kana' }; + if (han > 0 && han >= letters * 0.3) return { language: 'zh', script: 'han' }; + + const [script] = counts.reduce((best, entry) => (entry[1] > best[1] ? entry : best)); + + if (script in SCRIPT_LANGUAGE) return { language: SCRIPT_LANGUAGE[script], script }; + + const words = source.toLowerCase().split(/[^\p{L}\p{M}']+/u).filter(Boolean); + + if (script === 'cyrillic') { + const scores = scoreStopwords(words, CYRILLIC_LANGUAGES); + const ukMarkers = countMatches(source, /[іїєґ]/gi); + const ruMarkers = countMatches(source, /[ыэъё]/gi); + // Letters decide: the two alphabets differ in letters that occur in + // nearly every sentence. Function words only settle a text that shows + // neither set, and a text with no Russian-only letters is far more + // likely Ukrainian than the reverse, so that tie goes to Ukrainian. + if (ukMarkers !== ruMarkers) return { language: ukMarkers > ruMarkers ? 'uk' : 'ru', script }; + if (scores.uk !== scores.ru) return { language: scores.uk > scores.ru ? 'uk' : 'ru', script }; + return { language: ruMarkers > 0 ? 'ru' : 'uk', script }; + } + + const scores = scoreStopwords(words, LATIN_LANGUAGES); + const marked = pickByMarkers(source, LATIN_MARKERS); + // A characteristic letter outranks stopword counts unless another language + // clearly dominates the function words (a German text quoting "façade"). + if (marked && scores[marked] * 2 >= scores[bestOf(scores, marked)]) { + return { language: marked, script }; + } + return { language: bestOf(scores, 'en'), script }; +} + +/** + * Map a detected language onto the locales a voice list uses (`uk_UA`, + * `en_US`...). Returns the preferred locale prefixes in order. + * @param {string} language + * @returns {string[]} + */ +function localePrefixesForLanguage(language) { + const table = { + en: ['en_US', 'en_GB', 'en'], + uk: ['uk_UA', 'uk'], + ru: ['ru_RU', 'ru'], + de: ['de_DE', 'de'], + fr: ['fr_FR', 'fr_CA', 'fr'], + es: ['es_ES', 'es_MX', 'es'], + it: ['it_IT', 'it'], + pt: ['pt_BR', 'pt_PT', 'pt'], + pl: ['pl_PL', 'pl'], + nl: ['nl_NL', 'nl_BE', 'nl'], + cs: ['cs_CZ', 'cs'], + tr: ['tr_TR', 'tr'], + sv: ['sv_SE', 'sv'], + zh: ['zh_CN', 'zh_TW', 'zh_HK', 'zh'], + ja: ['ja_JP', 'ja'], + ko: ['ko_KR', 'ko'], + el: ['el_GR', 'el'], + ar: ['ar_001', 'ar_SA', 'ar'], + he: ['he_IL', 'he'], + th: ['th_TH', 'th'], + hi: ['hi_IN', 'hi'], + }; + return table[language] ?? [language]; +} + +/** + * Choose a voice for a language from a `say`-style voice list. + * Prefers an enhanced/premium variant of a matching voice, then any voice of + * the exact locale, then any voice of the language. Returns null when the + * list has no voice for that language. + * @param {string} language + * @param {ReadonlyArray<{ name: string, locale: string }>} voices + * @returns {string | null} + */ +export function pickVoiceForLanguage(language, voices) { + const prefixes = localePrefixesForLanguage(language); + for (const prefix of prefixes) { + const matching = voices.filter((voice) => voice.locale === prefix || voice.locale.startsWith(`${prefix}_`) || (prefix === language && voice.locale.startsWith(`${language}_`))); + if (matching.length === 0) continue; + const enhanced = matching.find((voice) => /\((Enhanced|Premium)\)/i.test(voice.name)); + return (enhanced ?? matching[0]).name; + } + return null; +} + +/** + * Language of a voice, from its locale (`uk_UA` → `uk`). + * @param {string | null | undefined} locale + * @returns {string | null} + */ +export function languageOfLocale(locale) { + if (typeof locale !== 'string' || !locale) return null; + return locale.split(/[_-]/)[0].toLowerCase(); +} diff --git a/packages/web/server/lib/tts/language-detect.test.js b/packages/web/server/lib/tts/language-detect.test.js new file mode 100644 index 00000000..a8c0f2c3 --- /dev/null +++ b/packages/web/server/lib/tts/language-detect.test.js @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { detectTextLanguage, languageOfLocale, pickVoiceForLanguage } from './language-detect.js'; + +describe('detectTextLanguage', () => { + it.each([ + ['en', 'The build is green and the tests pass, so you can merge this now.'], + ['uk', 'Привіт! Це тестове повідомлення, і воно написане українською мовою.'], + ['ru', 'Привет! Это тестовое сообщение, и оно написано на русском языке.'], + ['de', 'Die Änderung ist fertig und die Tests laufen ohne Fehler durch.'], + ['fr', 'La modification est prête et les tests passent sans erreur.'], + ['es', 'El cambio está listo y las pruebas pasan sin errores.'], + ['it', 'La modifica è pronta e i test passano senza errori.'], + ['pt', 'A alteração está pronta e os testes passam sem erros, você pode continuar.'], + ['pl', 'Zmiana jest gotowa i testy przechodzą bez błędów.'], + ['nl', 'De wijziging is klaar en de tests slagen zonder fouten.'], + ['cs', 'Změna je hotová a testy procházejí bez chyb.'], + ['tr', 'Değişiklik hazır ve testler hatasız geçiyor.'], + ['sv', 'Ändringen är klar och testerna går igenom utan fel.'], + ['zh', '修改已经完成，所有测试都通过了。'], + ['ja', '変更が完了し、すべてのテストに合格しました。'], + ['ko', '변경이 완료되었고 모든 테스트를 통과했습니다.'], + ])('detects %s', (language, text) => { + expect(detectTextLanguage(text).language).toBe(language); + }); + + it.each([ + ['uk', 'Готово. Запушено.'], + ['uk', 'Все ок'], + ['uk', 'Добре, давай так зробимо'], + ['ru', 'Хорошо, давай так и сделаем'], + ['ru', 'Готово, всё запушено.'], + ])('tells short %s phrases apart by letters', (language, text) => { + expect(detectTextLanguage(text).language).toBe(language); + }); + + it('falls back to English for text without letters', () => { + expect(detectTextLanguage('1234 ... !!!').language).toBe('en'); + expect(detectTextLanguage('').language).toBe('en'); + }); + + it('does not let a single quoted foreign word flip an English paragraph', () => { + const text = 'The façade of the building is the part that you see from the street, and it is not the same as the interior.'; + expect(detectTextLanguage(text).language).toBe('en'); + }); +}); + +describe('pickVoiceForLanguage', () => { + const voices = [ + { name: 'Samantha', locale: 'en_US' }, + { name: 'Daniel', locale: 'en_GB' }, + { name: 'Lesya', locale: 'uk_UA' }, + { name: 'Lesya (Enhanced)', locale: 'uk_UA' }, + { name: 'Milena', locale: 'ru_RU' }, + { name: 'Anna', locale: 'de_DE' }, + ]; + + it('prefers the enhanced variant of a matching voice', () => { + expect(pickVoiceForLanguage('uk', voices)).toBe('Lesya (Enhanced)'); + }); + + it('prefers the primary locale of a language', () => { + expect(pickVoiceForLanguage('en', voices)).toBe('Samantha'); + }); + + it('returns null when no voice speaks the language', () => { + expect(pickVoiceForLanguage('ja', voices)).toBeNull(); + }); +}); + +describe('languageOfLocale', () => { + it('reads the language subtag', () => { + expect(languageOfLocale('uk_UA')).toBe('uk'); + expect(languageOfLocale('en-GB')).toBe('en'); + expect(languageOfLocale(null)).toBeNull(); + }); +}); diff --git a/packages/web/server/lib/tts/routes.js b/packages/web/server/lib/tts/routes.js index 5d2c4618..2f2074ff 100644 --- a/packages/web/server/lib/tts/routes.js +++ b/packages/web/server/lib/tts/routes.js @@ -2,6 +2,8 @@ import express from 'express'; import { normalizeCustomOpenAIBaseURL } from './base-url.js'; import { summarizeText, sanitizeForTTS, sanitizeForNote } from '../text/summarization.js'; +import { detectTextLanguage, languageOfLocale, pickVoiceForLanguage } from './language-detect.js'; + export function registerTtsRoutes(app, { sayTTSCapability }) { let ttsModulePromise = null; const getTtsModule = async () => { @@ -154,7 +156,8 @@ export function registerTtsRoutes(app, { sayTTSCapability }) { // macOS 'say' command TTS speak endpoint app.post('/api/tts/say/speak', async (req, res) => { try { - const { text, voice = 'Samantha', rate = 200 } = req.body || {}; + const { text, rate = 200, language, languageSample } = req.body || {}; + let voice = typeof req.body?.voice === 'string' && req.body.voice.trim() ? req.body.voice.trim() : 'Samantha'; if (!text || typeof text !== 'string' || !text.trim()) { return res.status(400).json({ error: 'Text is required' }); @@ -164,6 +167,23 @@ export function registerTtsRoutes(app, { sayTTSCapability }) { if (process.platform !== 'darwin') { return res.status(503).json({ error: 'macOS say command not available on this platform' }); } + + // `language: 'auto'`: keep the chosen voice while it speaks the text's + // language, otherwise switch to an installed voice that does. A + // language with no installed voice keeps the chosen voice — say still + // reads the text, just with an accent — rather than failing. + let resolvedLanguage = null; + if (language === 'auto') { + const capability = await sayTTSCapability; + const voices = Array.isArray(capability?.voices) ? capability.voices : []; + const sample = typeof languageSample === 'string' && languageSample.trim() ? languageSample.slice(0, 4000) : text; + resolvedLanguage = detectTextLanguage(sample).language; + const chosen = voices.find((entry) => entry.name === voice); + if (languageOfLocale(chosen?.locale) !== resolvedLanguage) { + const match = pickVoiceForLanguage(resolvedLanguage, voices); + if (match) voice = match; + } + } const { exec } = await import('child_process'); const { promisify } = await import('util'); @@ -195,6 +215,8 @@ export function registerTtsRoutes(app, { sayTTSCapability }) { // Send audio response res.setHeader('Content-Type', 'audio/mp4'); + res.setHeader('X-Speech-Voice', voice); + if (resolvedLanguage) res.setHeader('X-Speech-Language', resolvedLanguage); res.setHeader('Content-Length', audioBuffer.length); res.send(audioBuffer); diff --git a/packages/web/server/lib/tts/routes.test.js b/packages/web/server/lib/tts/routes.test.js index f4940265..fce960a4 100644 --- a/packages/web/server/lib/tts/routes.test.js +++ b/packages/web/server/lib/tts/routes.test.js @@ -33,6 +33,32 @@ describe('tts routes', () => { }); }); + it('switches the say voice to the language of the text when asked to', async () => { + const capability = Promise.resolve({ + available: true, + voices: [ + { name: 'Samantha', locale: 'en_US' }, + { name: 'Lesya', locale: 'uk_UA' }, + { name: 'Lesya (Enhanced)', locale: 'uk_UA' }, + ], + }); + const app = createApp(capability); + const response = await request(app) + .post('/api/tts/say/speak') + .send({ text: 'Привіт! Це відповідь українською мовою, і вона досить довга.', voice: 'Samantha', language: 'auto' }); + + // On macOS the route synthesizes; elsewhere it refuses before running say. + // Either way the chosen voice must be the Ukrainian one when the platform + // allows the request to proceed. + if (process.platform === 'darwin') { + expect(response.status).toBe(200); + expect(response.headers['x-speech-voice']).toBe('Lesya (Enhanced)'); + expect(response.headers['x-speech-language']).toBe('uk'); + } else { + expect(response.status).toBe(503); + } + }); + it('returns local note fallback while model summarization is retired', async () => { const response = await request(createApp()) .post('/api/text/summarize')