diff --git a/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx b/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx index 8d3a1ac3..9abfeffe 100644 --- a/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx +++ b/packages/ui/src/components/sections/openchamber/VoiceSettings.tsx @@ -71,6 +71,7 @@ const LOCAL_STT_MODELS = [ interface DictationModelState { id: string; + description?: string; installed: boolean; downloading: boolean; downloadProgress: number | null; @@ -288,10 +289,32 @@ const KOKORO_VOICE_OPTIONS = [ const LOCAL_TTS_MODEL_ID = 'kokoro-en-v0_19'; -const LocalTtsModelStatus = () => { - const { t } = useI18n(); - const [model, setModel] = useState(null); - const [requesting, setRequesting] = useState(false); +const KOKORO_MULTI_LANG_MODEL_ID = 'kokoro-multi-lang-v1_1'; +// A few named speakers out of the 103 in the Chinese/English Kokoro build. +const KOKORO_MULTI_LANG_VOICE_OPTIONS = [ + { id: 0, label: 'Maple (af)' }, + { id: 1, label: 'Sol (af)' }, + { id: 2, label: 'Vale (bf)' }, + { id: 3, label: 'Xiaoxiao (zf)' }, + { id: 58, label: 'Yunxi (zm)' }, +]; + +interface LocalTtsVoiceOption { + modelId: string; + speakerId: number; + label: string; +} + +const localTtsVoiceKey = (modelId: string, speakerId: number): string => `${modelId}:${speakerId}`; + +/** + * Local TTS models as the server reports them, plus the actions Settings + * offers on them. Shared by the model list and the voice picker so both see + * the same install state. + */ +const useLocalTtsModels = () => { + const [models, setModels] = useState([]); + const [requestingId, setRequestingId] = useState(null); const refresh = useCallback(async () => { try { @@ -300,11 +323,8 @@ const LocalTtsModelStatus = () => { return; } const data = await response.json(); - const entry = Array.isArray(data?.ttsModels) - ? data.ttsModels.find((m: DictationModelState) => m.id === LOCAL_TTS_MODEL_ID) - : null; - if (entry) { - setModel(entry); + if (Array.isArray(data?.ttsModels)) { + setModels(data.ttsModels); } } catch { // Display-only status; keep the previous state on fetch failure. @@ -315,81 +335,118 @@ const LocalTtsModelStatus = () => { void refresh(); }, [refresh]); + const anyDownloading = models.some((model) => model.downloading); useEffect(() => { - if (!model?.downloading) { + if (!anyDownloading) { return; } const interval = setInterval(() => { void refresh(); }, 2000); return () => clearInterval(interval); - }, [model?.downloading, refresh]); + }, [anyDownloading, refresh]); - const request = async (method: 'POST' | 'DELETE') => { - setRequesting(true); + const request = useCallback(async (modelId: string, method: 'POST' | 'DELETE') => { + setRequestingId(modelId); try { const path = method === 'POST' - ? `/api/dictation/models/${LOCAL_TTS_MODEL_ID}/download` - : `/api/dictation/models/${LOCAL_TTS_MODEL_ID}`; + ? `/api/dictation/models/${modelId}/download` + : `/api/dictation/models/${modelId}`; await runtimeFetch(path, { method }); await refresh(); } catch { // Status refresh reports errors. } finally { - setRequesting(false); + setRequestingId(null); } - }; + }, [refresh]); - if (!model) { + return { models, requestingId, request, refresh }; +}; + +// Voices the picker offers: Kokoro speakers for the Kokoro models, one voice +// per installed Piper model. Only installed models (plus the default) appear, +// so a language model the server fetched on its own becomes selectable once +// it is on disk. +const buildLocalTtsVoiceOptions = (models: DictationModelState[]): LocalTtsVoiceOption[] => { + const options: LocalTtsVoiceOption[] = KOKORO_VOICE_OPTIONS.map((voice) => ({ + modelId: LOCAL_TTS_MODEL_ID, + speakerId: voice.id, + label: voice.label, + })); + for (const model of models) { + if (model.id === LOCAL_TTS_MODEL_ID || !model.installed) continue; + if (model.id === KOKORO_MULTI_LANG_MODEL_ID) { + for (const voice of KOKORO_MULTI_LANG_VOICE_OPTIONS) { + options.push({ modelId: model.id, speakerId: voice.id, label: `${voice.label} · Kokoro zh/en` }); + } + continue; + } + options.push({ modelId: model.id, speakerId: 0, label: model.description ?? model.id }); + } + return options; +}; + +const LocalTtsModelStatus = ({ models, requestingId, request }: ReturnType) => { + const { t } = useI18n(); + + // The default English model is always listed; language models the server + // fetched on its own appear once they are installed or downloading, so + // the list shows what is on disk rather than the whole catalog. + const visible = models.filter((model) => model.id === LOCAL_TTS_MODEL_ID || model.installed || model.downloading); + if (visible.length === 0) { return null; } return ( -
- Kokoro - 305 MB - {model.installed ? ( - <> - - - - ) : model.downloading ? ( - - - - {typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''} - - - ) : ( - - )} - {model.downloadError ? ( - {model.downloadError} - ) : null} +
+ {visible.map((model) => ( +
+ {model.description ?? model.id} + {model.installed ? ( + <> + + + + ) : model.downloading ? ( + + + + {typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''} + + + ) : ( + + )} + {model.downloadError ? ( + {model.downloadError} + ) : null} +
+ ))}
); }; @@ -424,6 +481,12 @@ export const VoiceSettings: React.FC = () => { const sayVoice = useConfigStore((state) => state.sayVoice); const setSayVoice = useConfigStore((state) => state.setSayVoice); const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId); + const localTtsModelId = useConfigStore((state) => state.localTtsModelId); + const setLocalTtsModelId = useConfigStore((state) => state.setLocalTtsModelId); + const localTtsModels = useLocalTtsModels(); + const localTtsVoiceOptions = useMemo(() => buildLocalTtsVoiceOptions(localTtsModels.models), [localTtsModels.models]); + const ttsFollowTextLanguage = useConfigStore((state) => state.ttsFollowTextLanguage); + const setTtsFollowTextLanguage = useConfigStore((state) => state.setTtsFollowTextLanguage); const setLocalTtsVoiceId = useConfigStore((state) => state.setLocalTtsVoiceId); const { speak: speakLocalTts, stop: stopLocalTts, isPlaying: isLocalTtsPlaying, error: localTtsError } = useLocalTTS(); @@ -432,13 +495,14 @@ export const VoiceSettings: React.FC = () => { stopLocalTts(); return; } - const voiceLabel = KOKORO_VOICE_OPTIONS.find((v) => v.id === localTtsVoiceId)?.label + const voiceLabel = localTtsVoiceOptions.find((v) => v.modelId === localTtsModelId && v.speakerId === localTtsVoiceId)?.label ?? String(localTtsVoiceId); void speakLocalTts(t('settings.voice.page.preview.voiceLine', { voiceName: voiceLabel }), { + model: localTtsModelId, speakerId: localTtsVoiceId, speed: useConfigStore.getState().speechRate, }); - }, [isLocalTtsPlaying, localTtsVoiceId, speakLocalTts, stopLocalTts, t]); + }, [isLocalTtsPlaying, localTtsModelId, localTtsVoiceId, localTtsVoiceOptions, speakLocalTts, stopLocalTts, t]); const browserVoice = useConfigStore((state) => state.browserVoice); const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice); const openaiVoice = useConfigStore((state) => state.openaiVoice); @@ -959,24 +1023,39 @@ export const VoiceSettings: React.FC = () => { )} {/* Local (Kokoro) TTS model status */} - {voiceProvider === 'local' && } + {voiceProvider === 'local' && } + + {(voiceProvider === 'local' || voiceProvider === 'say') && ( + + )} {/* Voice Selection */} {voiceProvider === 'local' && ( <> diff --git a/packages/ui/src/hooks/useLocalTTS.ts b/packages/ui/src/hooks/useLocalTTS.ts index 307356a2..0efc1a46 100644 --- a/packages/ui/src/hooks/useLocalTTS.ts +++ b/packages/ui/src/hooks/useLocalTTS.ts @@ -14,10 +14,18 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { runtimeFetch } from '@/lib/runtime-fetch'; export interface LocalTTSSpeakOptions { - /** Kokoro speaker id (0-10) */ + /** Catalog id of the local model to use; defaults to the server's default model. */ + model?: string; + /** Speaker id within the model (Kokoro voices; Piper models have one) */ speakerId?: number; /** Playback speed multiplier (1.0 = normal) */ speed?: number; + /** + * `'auto'`: the server picks a model and voice for the text's language. + * The language is judged on the whole message, not on each chunk sent for + * synthesis, so a short chunk cannot flip the voice mid-reply. + */ + language?: 'auto'; onStart?: () => void; onEnd?: () => void; onError?: (error: string) => void; @@ -35,6 +43,8 @@ export interface UseLocalTTSReturn { /** Target chunk size: big enough to amortize requests, small enough for low latency. */ const MIN_CHUNK_CHARS = 60; const MAX_CHUNK_CHARS = 400; +// Enough of the message for language detection to see whole sentences. +const LANGUAGE_SAMPLE_CHARS = 2000; /** * Split text into sentence-aligned chunks for pipelined synthesis. @@ -170,6 +180,7 @@ export function useLocalTTS(): UseLocalTTSReturn { const session: PlaybackSession = { cancelled: false, abort: new AbortController() }; sessionRef.current = session; + const languageSample = options?.language === 'auto' ? text.slice(0, LANGUAGE_SAMPLE_CHARS) : undefined; const fetchChunk = async (chunk: string): Promise => { const response = await runtimeFetch('/api/dictation/tts/speak', { @@ -177,8 +188,11 @@ export function useLocalTTS(): UseLocalTTSReturn { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: chunk, + model: options?.model, ...(typeof options?.speakerId === 'number' ? { speakerId: options.speakerId } : {}), ...(typeof options?.speed === 'number' ? { speed: options.speed } : {}), + language: options?.language, + languageSample, }), signal: session.abort.signal, }); diff --git a/packages/ui/src/hooks/useMessageTTS.ts b/packages/ui/src/hooks/useMessageTTS.ts index 88e10615..4950e320 100644 --- a/packages/ui/src/hooks/useMessageTTS.ts +++ b/packages/ui/src/hooks/useMessageTTS.ts @@ -61,6 +61,8 @@ export function useMessageTTS(): UseMessageTTSReturn { const speechVolume = useConfigStore((state) => state.speechVolume); const sayVoice = useConfigStore((state) => state.sayVoice); const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId); + const localTtsModelId = useConfigStore((state) => state.localTtsModelId); + const ttsFollowTextLanguage = useConfigStore((state) => state.ttsFollowTextLanguage); const browserVoice = useConfigStore((state) => state.browserVoice); const openaiVoice = useConfigStore((state) => state.openaiVoice); const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice); @@ -135,8 +137,10 @@ export function useMessageTTS(): UseMessageTTSReturn { }); } else if (voiceProvider === 'local') { await speakLocalTTS(sanitizedText, { + model: localTtsModelId, speakerId: localTtsVoiceId, speed: speechRate, + language: ttsFollowTextLanguage ? 'auto' : undefined, onEnd: () => setIsPlaying(false), onError: () => setIsPlaying(false), }); @@ -145,6 +149,7 @@ export function useMessageTTS(): UseMessageTTSReturn { await speakSayTTS(sanitizedText, { voice: sayVoice, rate: wordsPerMinute, + language: ttsFollowTextLanguage ? 'auto' : undefined, onEnd: () => setIsPlaying(false), onError: () => setIsPlaying(false), }); @@ -187,6 +192,8 @@ export function useMessageTTS(): UseMessageTTSReturn { speakSayTTS, speakLocalTTS, localTtsVoiceId, + localTtsModelId, + ttsFollowTextLanguage, stop, ]); diff --git a/packages/ui/src/hooks/useSayTTS.ts b/packages/ui/src/hooks/useSayTTS.ts index c00353b1..f9108e6d 100644 --- a/packages/ui/src/hooks/useSayTTS.ts +++ b/packages/ui/src/hooks/useSayTTS.ts @@ -105,6 +105,8 @@ interface SpeakOptions { voice?: string; /** Speech rate in words per minute (defaults to 200) */ rate?: number; + /** `'auto'`: the server switches to a voice that speaks the text's language. */ + language?: 'auto'; /** Callback when playback starts */ onStart?: () => void; /** Callback when playback ends */ @@ -229,6 +231,7 @@ export function useSayTTS(options: UseSayTTSOptions = {}): UseSayTTSReturn { text: text.trim(), voice: options?.voice || 'Samantha', rate: options?.rate || 200, + language: options?.language, }), signal: abortControllerRef.current.signal, }); diff --git a/packages/ui/src/lib/i18n/messages/de.settings.ts b/packages/ui/src/lib/i18n/messages/de.settings.ts index 75d596f9..86dc0985 100644 --- a/packages/ui/src/lib/i18n/messages/de.settings.ts +++ b/packages/ui/src/lib/i18n/messages/de.settings.ts @@ -1795,7 +1795,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Server', 'settings.voice.page.provider.local': 'Lokal', 'settings.voice.page.tooltip.sttLocal': 'On-device Transkription auf dem OpenChamber-Server. Modelle werden automatisch heruntergeladen; kein API-Schlüssel erforderlich.', - 'settings.voice.page.tooltip.localTts': 'On-device Synthese auf dem OpenChamber-Server (Kokoro, Englisch). Das Modell wird automatisch heruntergeladen; kein API-Schlüssel erforderlich.', + 'settings.voice.page.tooltip.localTts': 'On-Device-Synthese auf dem OpenChamber-Server (Kokoro für Englisch; Modelle für andere Sprachen werden beim ersten Einsatz geladen). Kein API-Schlüssel nötig.', + 'settings.voice.page.field.followTextLanguage': 'Stimme an die Sprache des Textes anpassen', + 'settings.voice.page.field.followTextLanguageAria': 'Stimme an die Sprache des Textes anpassen', + 'settings.voice.page.field.followTextLanguageInfo': 'Ist eine Antwort in einer anderen Sprache, wird eine Stimme für diese Sprache verwendet: eine passende macOS-Stimme oder ein lokales Modell, das beim ersten Einsatz geladen wird.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (Englisch)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 europäische Sprachen)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (mehrsprachig)', diff --git a/packages/ui/src/lib/i18n/messages/en.settings.ts b/packages/ui/src/lib/i18n/messages/en.settings.ts index 95620f19..e0c6468f 100644 --- a/packages/ui/src/lib/i18n/messages/en.settings.ts +++ b/packages/ui/src/lib/i18n/messages/en.settings.ts @@ -1862,7 +1862,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Server', 'settings.voice.page.provider.local': 'Local', 'settings.voice.page.tooltip.sttLocal': 'On-device transcription on the OpenChamber server. Models download automatically; no API key needed.', - 'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro, English). The model downloads automatically; no API key needed.', + 'settings.voice.page.tooltip.localTts': 'On-device synthesis on the OpenChamber server (Kokoro for English; models for other languages download on first use). No API key needed.', + 'settings.voice.page.field.followTextLanguage': 'Match the voice to the language of the text', + 'settings.voice.page.field.followTextLanguageAria': 'Match the voice to the language of the text', + 'settings.voice.page.field.followTextLanguageInfo': 'When a reply is in another language, a voice for that language is used: a matching macOS voice, or a local model that downloads on first use.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (English)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 European languages)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingual)', diff --git a/packages/ui/src/lib/i18n/messages/es.settings.ts b/packages/ui/src/lib/i18n/messages/es.settings.ts index fa40f214..b72f9e26 100644 --- a/packages/ui/src/lib/i18n/messages/es.settings.ts +++ b/packages/ui/src/lib/i18n/messages/es.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { "settings.voice.page.provider.server": "Servidor", "settings.voice.page.provider.local": "Local", "settings.voice.page.tooltip.sttLocal": "Transcripción local en el servidor de OpenChamber. Los modelos se descargan automáticamente; no se necesita clave de API.", - "settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro, inglés). El modelo se descarga automáticamente; no se necesita clave de API.", + "settings.voice.page.tooltip.localTts": "Síntesis local en el servidor de OpenChamber (Kokoro para inglés; los modelos de otros idiomas se descargan en el primer uso). No requiere clave de API.", + "settings.voice.page.field.followTextLanguage": "Ajustar la voz al idioma del texto", + "settings.voice.page.field.followTextLanguageAria": "Ajustar la voz al idioma del texto", + "settings.voice.page.field.followTextLanguageInfo": "Si una respuesta está en otro idioma, se usa una voz para ese idioma: una voz de macOS adecuada o un modelo local que se descarga en el primer uso.", "settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglés)", "settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeos)", "settings.voice.page.stt.model.whisperBase": "Whisper base (multilingüe)", diff --git a/packages/ui/src/lib/i18n/messages/fr.settings.ts b/packages/ui/src/lib/i18n/messages/fr.settings.ts index 432d536d..ae7d5d4e 100644 --- a/packages/ui/src/lib/i18n/messages/fr.settings.ts +++ b/packages/ui/src/lib/i18n/messages/fr.settings.ts @@ -1757,7 +1757,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Serveur', 'settings.voice.page.provider.local': 'Local', 'settings.voice.page.tooltip.sttLocal': 'Transcription locale sur le serveur OpenChamber. Les modèles se téléchargent automatiquement ; aucune clé d\'API requise.', - 'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro, anglais). Le modèle se télécharge automatiquement ; aucune clé d’API requise.', + 'settings.voice.page.tooltip.localTts': 'Synthèse locale sur le serveur OpenChamber (Kokoro pour l’anglais ; les modèles des autres langues sont téléchargés à la première utilisation). Aucune clé API requise.', + 'settings.voice.page.field.followTextLanguage': 'Adapter la voix à la langue du texte', + 'settings.voice.page.field.followTextLanguageAria': 'Adapter la voix à la langue du texte', + 'settings.voice.page.field.followTextLanguageInfo': 'Si une réponse est dans une autre langue, une voix pour cette langue est utilisée : une voix macOS adaptée ou un modèle local téléchargé à la première utilisation.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (anglais)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 langues européennes)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (multilingue)', diff --git a/packages/ui/src/lib/i18n/messages/ja.settings.ts b/packages/ui/src/lib/i18n/messages/ja.settings.ts index 87b35b08..a68f0fc7 100644 --- a/packages/ui/src/lib/i18n/messages/ja.settings.ts +++ b/packages/ui/src/lib/i18n/messages/ja.settings.ts @@ -1872,7 +1872,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'サーバー', 'settings.voice.page.provider.local': 'ローカル', 'settings.voice.page.tooltip.sttLocal': 'OpenChamber サーバー上でローカルに文字起こしします。モデルは自動でダウンロードされ、API キーは不要です。', - 'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します(Kokoro、英語)。モデルは自動でダウンロードされ、API キーは不要です。', + 'settings.voice.page.tooltip.localTts': 'OpenChamber サーバー上でローカルに音声合成します(英語は Kokoro、他の言語のモデルは初回使用時にダウンロード)。API キーは不要です。', + 'settings.voice.page.field.followTextLanguage': 'テキストの言語に合わせて音声を選ぶ', + 'settings.voice.page.field.followTextLanguageAria': 'テキストの言語に合わせて音声を選ぶ', + 'settings.voice.page.field.followTextLanguageInfo': '返答が別の言語の場合、その言語の音声を使います。対応する macOS の音声、または初回使用時にダウンロードされるローカルモデルです。', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英語)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(ヨーロッパ25言語)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base(多言語)', diff --git a/packages/ui/src/lib/i18n/messages/ko.settings.ts b/packages/ui/src/lib/i18n/messages/ko.settings.ts index 08ac89c7..4fdb93af 100644 --- a/packages/ui/src/lib/i18n/messages/ko.settings.ts +++ b/packages/ui/src/lib/i18n/messages/ko.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': '서버', 'settings.voice.page.provider.local': '로컬', 'settings.voice.page.tooltip.sttLocal': 'OpenChamber 서버에서 로컬로 변환합니다. 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.', - 'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(Kokoro, 영어). 모델은 자동으로 다운로드되며 API 키가 필요 없습니다.', + 'settings.voice.page.tooltip.localTts': 'OpenChamber 서버에서 로컬로 음성을 합성합니다(영어는 Kokoro, 다른 언어 모델은 처음 사용할 때 다운로드). API 키가 필요 없습니다.', + 'settings.voice.page.field.followTextLanguage': '텍스트 언어에 맞는 음성 사용', + 'settings.voice.page.field.followTextLanguageAria': '텍스트 언어에 맞는 음성 사용', + 'settings.voice.page.field.followTextLanguageInfo': '응답이 다른 언어이면 해당 언어의 음성을 사용합니다. 일치하는 macOS 음성 또는 처음 사용할 때 다운로드되는 로컬 모델입니다.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (영어)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (유럽 25개 언어)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (다국어)', diff --git a/packages/ui/src/lib/i18n/messages/pl.settings.ts b/packages/ui/src/lib/i18n/messages/pl.settings.ts index ac8d5357..f8272cc1 100644 --- a/packages/ui/src/lib/i18n/messages/pl.settings.ts +++ b/packages/ui/src/lib/i18n/messages/pl.settings.ts @@ -2176,7 +2176,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Serwer', 'settings.voice.page.provider.local': 'Lokalny', 'settings.voice.page.tooltip.sttLocal': 'Transkrypcja lokalna na serwerze OpenChamber. Modele pobierają się automatycznie; klucz API nie jest potrzebny.', - 'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro, angielski). Model pobiera się automatycznie; klucz API nie jest potrzebny.', + 'settings.voice.page.tooltip.localTts': 'Lokalna synteza na serwerze OpenChamber (Kokoro dla angielskiego; modele innych języków pobierane przy pierwszym użyciu). Klucz API nie jest potrzebny.', + 'settings.voice.page.field.followTextLanguage': 'Dopasuj głos do języka tekstu', + 'settings.voice.page.field.followTextLanguageAria': 'Dopasuj głos do języka tekstu', + 'settings.voice.page.field.followTextLanguageInfo': 'Gdy odpowiedź jest w innym języku, używany jest głos dla tego języka: pasujący głos macOS albo lokalny model pobierany przy pierwszym użyciu.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (angielski)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 języków europejskich)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (wielojęzyczny)', diff --git a/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts b/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts index fb5fa2c1..3f2827c5 100644 --- a/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts +++ b/packages/ui/src/lib/i18n/messages/pt-BR.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { "settings.voice.page.provider.server": "Servidor", "settings.voice.page.provider.local": "Local", "settings.voice.page.tooltip.sttLocal": "Transcrição local no servidor do OpenChamber. Os modelos são baixados automaticamente; não é necessária chave de API.", - "settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro, inglês). O modelo é baixado automaticamente; não é necessária chave de API.", + "settings.voice.page.tooltip.localTts": "Síntese local no servidor do OpenChamber (Kokoro para inglês; modelos de outros idiomas são baixados no primeiro uso). Não requer chave de API.", + "settings.voice.page.field.followTextLanguage": "Ajustar a voz ao idioma do texto", + "settings.voice.page.field.followTextLanguageAria": "Ajustar a voz ao idioma do texto", + "settings.voice.page.field.followTextLanguageInfo": "Se uma resposta estiver em outro idioma, uma voz desse idioma é usada: uma voz do macOS correspondente ou um modelo local baixado no primeiro uso.", "settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (inglês)", "settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 idiomas europeus)", "settings.voice.page.stt.model.whisperBase": "Whisper base (multilíngue)", diff --git a/packages/ui/src/lib/i18n/messages/tr.settings.ts b/packages/ui/src/lib/i18n/messages/tr.settings.ts index c4ce2859..ead5621c 100644 --- a/packages/ui/src/lib/i18n/messages/tr.settings.ts +++ b/packages/ui/src/lib/i18n/messages/tr.settings.ts @@ -1787,7 +1787,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': 'Sunucu', 'settings.voice.page.provider.local': 'Yerel', 'settings.voice.page.tooltip.sttLocal': 'OpenChamber sunucusunda cihaz üstü transkripsiyon. Modeller otomatik indirilir; API anahtarı gerekmez.', - 'settings.voice.page.tooltip.localTts': 'OpenChamber sunucusunda cihaz üstü sentez (Kokoro, İngilizce). Model otomatik indirilir; API anahtarı gerekmez.', + 'settings.voice.page.tooltip.localTts': 'OpenChamber sunucusunda yerel sentez (İngilizce için Kokoro; diğer dillerin modelleri ilk kullanımda indirilir). API anahtarı gerekmez.', + 'settings.voice.page.field.followTextLanguage': 'Sesi metnin diline göre seç', + 'settings.voice.page.field.followTextLanguageAria': 'Sesi metnin diline göre seç', + 'settings.voice.page.field.followTextLanguageInfo': 'Yanıt başka bir dildeyse o dil için bir ses kullanılır: uygun bir macOS sesi veya ilk kullanımda indirilen yerel bir model.', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2 (İngilizce)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3 (25 Avrupa dili)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base (çok dilli)', diff --git a/packages/ui/src/lib/i18n/messages/uk.settings.ts b/packages/ui/src/lib/i18n/messages/uk.settings.ts index d65facf2..f3d88411 100644 --- a/packages/ui/src/lib/i18n/messages/uk.settings.ts +++ b/packages/ui/src/lib/i18n/messages/uk.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { "settings.voice.page.provider.server": "Сервер", "settings.voice.page.provider.local": "Локальний", "settings.voice.page.tooltip.sttLocal": "Локальна розшифровка на сервері OpenChamber. Моделі завантажуються автоматично; ключ API не потрібен.", - "settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro, англійська). Модель завантажується автоматично; ключ API не потрібен.", + "settings.voice.page.tooltip.localTts": "Локальний синтез на сервері OpenChamber (Kokoro для англійської; моделі для інших мов завантажуються при першому використанні). Ключ API не потрібен.", + "settings.voice.page.field.followTextLanguage": "Підбирати голос під мову тексту", + "settings.voice.page.field.followTextLanguageAria": "Підбирати голос під мову тексту", + "settings.voice.page.field.followTextLanguageInfo": "Якщо відповідь іншою мовою, використовується голос цієї мови: відповідний голос macOS або локальна модель, яка завантажується при першому використанні.", "settings.voice.page.stt.model.parakeetV2": "Parakeet v2 (англійська)", "settings.voice.page.stt.model.parakeetV3": "Parakeet v3 (25 європейських мов)", "settings.voice.page.stt.model.whisperBase": "Whisper base (мультимовна)", diff --git a/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts b/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts index c7f1f622..c0eef58a 100644 --- a/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts +++ b/packages/ui/src/lib/i18n/messages/zh-CN.settings.ts @@ -1839,7 +1839,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': '服务器', 'settings.voice.page.provider.local': '本地', 'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 服务器上本地转写。模型自动下载,无需 API 密钥。', - 'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音(Kokoro,英语)。模型自动下载,无需 API 密钥。', + 'settings.voice.page.tooltip.localTts': '在 OpenChamber 服务器上本地合成语音(英语使用 Kokoro;其他语言的模型在首次使用时下载)。无需 API 密钥。', + 'settings.voice.page.field.followTextLanguage': '根据文本语言匹配语音', + 'settings.voice.page.field.followTextLanguageAria': '根据文本语言匹配语音', + 'settings.voice.page.field.followTextLanguageInfo': '当回复使用其他语言时,将使用该语言的语音:匹配的 macOS 语音,或首次使用时下载的本地模型。', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英语)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(25 种欧洲语言)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base(多语言)', diff --git a/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts b/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts index e24521ae..c0639769 100644 --- a/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts +++ b/packages/ui/src/lib/i18n/messages/zh-TW.settings.ts @@ -1746,7 +1746,10 @@ export const settingsDict = { 'settings.voice.page.provider.server': '伺服器', 'settings.voice.page.provider.local': '本機', 'settings.voice.page.tooltip.sttLocal': '在 OpenChamber 伺服器上本機轉寫。模型會自動下載,無需 API 金鑰。', - 'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音(Kokoro,英文)。模型會自動下載,無需 API 金鑰。', + 'settings.voice.page.tooltip.localTts': '在 OpenChamber 伺服器上本機合成語音(英文使用 Kokoro;其他語言的模型在首次使用時下載)。不需要 API 金鑰。', + 'settings.voice.page.field.followTextLanguage': '依文字語言選擇語音', + 'settings.voice.page.field.followTextLanguageAria': '依文字語言選擇語音', + 'settings.voice.page.field.followTextLanguageInfo': '當回覆使用其他語言時,會使用該語言的語音:相符的 macOS 語音,或首次使用時下載的本機模型。', 'settings.voice.page.stt.model.parakeetV2': 'Parakeet v2(英文)', 'settings.voice.page.stt.model.parakeetV3': 'Parakeet v3(25 種歐洲語言)', 'settings.voice.page.stt.model.whisperBase': 'Whisper base(多語言)', diff --git a/packages/ui/src/stores/useConfigStore.ts b/packages/ui/src/stores/useConfigStore.ts index b4a370ca..66f1487e 100644 --- a/packages/ui/src/stores/useConfigStore.ts +++ b/packages/ui/src/stores/useConfigStore.ts @@ -1067,6 +1067,10 @@ interface ConfigStore { sayVoice: string; browserVoice: string; localTtsVoiceId: number; + /** Local TTS model the chosen voice belongs to (catalog id). */ + localTtsModelId: string; + /** Local and macOS voices follow the language of the text being read. */ + ttsFollowTextLanguage: boolean; openaiVoice: string; openaiApiKey: string; openaiCompatibleUrl: string; @@ -1094,6 +1098,8 @@ interface ConfigStore { setSayVoice: (voice: string) => void; setBrowserVoice: (voice: string) => void; setLocalTtsVoiceId: (voiceId: number) => void; + setLocalTtsModelId: (modelId: string) => void; + setTtsFollowTextLanguage: (enabled: boolean) => void; setOpenaiVoice: (voice: string) => void; setOpenaiApiKey: (apiKey: string) => void; setOpenaiCompatibleUrl: (url: string) => void; @@ -1277,6 +1283,21 @@ export const useConfigStore = create()( } return 0; })(), + localTtsModelId: (() => { + if (typeof window !== 'undefined') { + const saved = localStorage.getItem('localTtsModelId'); + if (saved) return saved; + } + return 'kokoro-en-v0_19'; + })(), + + ttsFollowTextLanguage: (() => { + if (typeof window !== 'undefined') { + const saved = localStorage.getItem('ttsFollowTextLanguage'); + if (saved !== null) return saved === 'true'; + } + return true; + })(), // Browser voice - load from localStorage or default to empty (auto-select) browserVoice: (() => { if (typeof window !== 'undefined') { @@ -2962,6 +2983,20 @@ export const useConfigStore = create()( } }, + setLocalTtsModelId: (modelId: string) => { + set({ localTtsModelId: modelId }); + if (typeof window !== 'undefined') { + localStorage.setItem('localTtsModelId', modelId); + } + }, + + setTtsFollowTextLanguage: (enabled: boolean) => { + set({ ttsFollowTextLanguage: enabled }); + if (typeof window !== 'undefined') { + localStorage.setItem('ttsFollowTextLanguage', String(enabled)); + } + }, + setBrowserVoice: (voice: string) => { set({ browserVoice: voice }); if (typeof window !== 'undefined') { diff --git a/packages/web/server/lib/dictation/DOCUMENTATION.md b/packages/web/server/lib/dictation/DOCUMENTATION.md index e66af116..49e38aae 100644 --- a/packages/web/server/lib/dictation/DOCUMENTATION.md +++ b/packages/web/server/lib/dictation/DOCUMENTATION.md @@ -11,12 +11,25 @@ live transcript costs O(n^2) work for a result the final decode replaces. The composer shows no text while recording and inserts the full transcript on stop. -Local TTS (Kokoro via sherpa-onnx OfflineTts) runs in the same worker process -and is exposed as `POST /api/dictation/tts/speak` (JSON `{text, speakerId?, -speed?, model?}` → WAV bytes; 503 with `reasonCode` while the model is -downloading). TTS models live in the same catalog/downloader as STT models -(`local/model-catalog.js` `LOCAL_TTS_MODEL_CATALOG`) and are managed by the -same status/download/delete routes. +Local TTS (Kokoro and Piper/VITS via sherpa-onnx OfflineTts) runs in the same +worker process and is exposed as `POST /api/dictation/tts/speak` (JSON +`{text, speakerId?, speed?, model?, language?, languageSample?}` → WAV bytes; 503 with +`reasonCode` while the model is downloading). TTS models live in the same +catalog/downloader as STT models (`local/model-catalog.js` +`LOCAL_TTS_MODEL_CATALOG`) and are managed by the same status/download/delete +routes. + +Each TTS catalog entry declares the `languages` it speaks. With +`language: 'auto'` the service detects the language of `languageSample` — the +whole message the chunk belongs to, sent by the client with every chunk — or +of `text` when no sample is given +(`../tts/language-detect.js`, script plus function-word scoring, no +dependencies) and keeps the caller's model when it speaks that language; +otherwise it switches to the catalog model for the language, downloading it on +first use like any other model, and starts from that model's default speaker +(`defaultSpeakerByLanguage`) instead of the caller's speaker id. A language no +catalog model covers keeps the caller's model, so text is always spoken. The +response carries `X-Speech-Model` and `X-Speech-Language`. ## Ownership diff --git a/packages/web/server/lib/dictation/local/model-catalog.js b/packages/web/server/lib/dictation/local/model-catalog.js index 29ea524b..16af62c9 100644 --- a/packages/web/server/lib/dictation/local/model-catalog.js +++ b/packages/web/server/lib/dictation/local/model-catalog.js @@ -68,9 +68,19 @@ export const LOCAL_STT_MODEL_CATALOG = { * Local text-to-speech models (sherpa-onnx OfflineTts). Downloaded and * managed through the same pipeline as the STT models. */ +/** + * Local text-to-speech models (sherpa-onnx OfflineTts). Downloaded and + * managed through the same pipeline as the STT models. + * + * `languages` lists the languages a model speaks well; the speech service + * uses it to pick a model for the language a text is written in. Kokoro + * models carry speaker ids (`voices`); a Piper model is one voice for one + * language. `lexicon` entries are joined with commas for sherpa-onnx. + */ export const LOCAL_TTS_MODEL_CATALOG = { 'kokoro-en-v0_19': { type: 'kokoro', + languages: ['en'], archiveUrl: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2', extractedDir: 'kokoro-en-v0_19', @@ -82,6 +92,188 @@ export const LOCAL_TTS_MODEL_CATALOG = { }, description: 'Kokoro TTS (English, natural voices)', }, + 'kokoro-multi-lang-v1_1': { + type: 'kokoro', + languages: ['zh', 'en'], + // sherpa-onnx wires this Kokoro build for Chinese and English only; + // speakers 0-2 are English, 3-102 Chinese. + defaultSpeakerByLanguage: { en: 0, zh: 3 }, + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_1.tar.bz2', + extractedDir: 'kokoro-multi-lang-v1_1', + files: { + model: 'model.onnx', + voices: 'voices.bin', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + lexiconEnglish: 'lexicon-us-en.txt', + lexiconChinese: 'lexicon-zh.txt', + }, + lexicon: ['lexiconEnglish', 'lexiconChinese'], + description: 'Kokoro TTS (Chinese and English, 103 voices)', + }, + // The larger `ukrainian_tts-medium` build is a character-level model + // (`phoneme_type: text`); sherpa-onnx phonemizes every Piper model through + // espeak-ng, which turns that one into noise. `vits-coqui-uk-mai` sounds + // better but reads Cyrillic only and drops every Latin word (file names, + // product names), which is unusable in a coding chat. Lada is an espeak + // model: small, but it reads mixed text. + 'piper-uk_UA-lada-x_low': { + type: 'vits', + languages: ['uk'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-uk_UA-lada-x_low.tar.bz2', + extractedDir: 'vits-piper-uk_UA-lada-x_low', + files: { + model: 'uk_UA-lada-x_low.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Ukrainian)', + }, + 'piper-de_DE-thorsten-medium': { + type: 'vits', + languages: ['de'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-thorsten-medium.tar.bz2', + extractedDir: 'vits-piper-de_DE-thorsten-medium', + files: { + model: 'de_DE-thorsten-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (German)', + }, + 'piper-fr_FR-siwis-medium': { + type: 'vits', + languages: ['fr'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-fr_FR-siwis-medium.tar.bz2', + extractedDir: 'vits-piper-fr_FR-siwis-medium', + files: { + model: 'fr_FR-siwis-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (French)', + }, + 'piper-es_ES-davefx-medium': { + type: 'vits', + languages: ['es'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-es_ES-davefx-medium.tar.bz2', + extractedDir: 'vits-piper-es_ES-davefx-medium', + files: { + model: 'es_ES-davefx-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Spanish)', + }, + 'piper-it_IT-paola-medium': { + type: 'vits', + languages: ['it'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-it_IT-paola-medium.tar.bz2', + extractedDir: 'vits-piper-it_IT-paola-medium', + files: { + model: 'it_IT-paola-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Italian)', + }, + 'piper-pt_BR-faber-medium': { + type: 'vits', + languages: ['pt'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-pt_BR-faber-medium.tar.bz2', + extractedDir: 'vits-piper-pt_BR-faber-medium', + files: { + model: 'pt_BR-faber-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Portuguese (Brazil))', + }, + 'piper-pl_PL-gosia-medium': { + type: 'vits', + languages: ['pl'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-pl_PL-gosia-medium.tar.bz2', + extractedDir: 'vits-piper-pl_PL-gosia-medium', + files: { + model: 'pl_PL-gosia-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Polish)', + }, + 'piper-ru_RU-irina-medium': { + type: 'vits', + languages: ['ru'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-ru_RU-irina-medium.tar.bz2', + extractedDir: 'vits-piper-ru_RU-irina-medium', + files: { + model: 'ru_RU-irina-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Russian)', + }, + 'piper-nl_NL-pim-medium': { + type: 'vits', + languages: ['nl'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-nl_NL-pim-medium.tar.bz2', + extractedDir: 'vits-piper-nl_NL-pim-medium', + files: { + model: 'nl_NL-pim-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Dutch)', + }, + 'piper-cs_CZ-jirka-medium': { + type: 'vits', + languages: ['cs'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-cs_CZ-jirka-medium.tar.bz2', + extractedDir: 'vits-piper-cs_CZ-jirka-medium', + files: { + model: 'cs_CZ-jirka-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Czech)', + }, + 'piper-tr_TR-dfki-medium': { + type: 'vits', + languages: ['tr'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-tr_TR-dfki-medium.tar.bz2', + extractedDir: 'vits-piper-tr_TR-dfki-medium', + files: { + model: 'tr_TR-dfki-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Turkish)', + }, + 'piper-sv_SE-nst-medium': { + type: 'vits', + languages: ['sv'], + archiveUrl: + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-sv_SE-nst-medium.tar.bz2', + extractedDir: 'vits-piper-sv_SE-nst-medium', + files: { + model: 'sv_SE-nst-medium.onnx', + tokens: 'tokens.txt', + espeakData: 'espeak-ng-data', + }, + description: 'Piper TTS (Swedish)', + }, }; export const DEFAULT_LOCAL_STT_MODEL = 'parakeet-tdt-0.6b-v2-int8'; @@ -131,6 +323,34 @@ export function getLocalSttModelSpec(modelId) { }; } +/** + * The local TTS model to use for a language, preferring the model the user + * selected when it speaks that language. Returns null when no catalog model + * covers the language, in which case callers keep the selected model. + * @param {string} language BCP-47 primary subtag (`uk`, `zh`...) + * @param {string} [preferredModelId] + * @returns {string | null} + */ +export function resolveLocalTtsModelForLanguage(language, preferredModelId) { + const speaks = (modelId) => LOCAL_TTS_MODEL_CATALOG[modelId]?.languages?.includes(language) === true; + if (preferredModelId && speaks(preferredModelId)) return preferredModelId; + const candidate = LOCAL_TTS_MODEL_IDS.find(speaks); + return candidate ?? null; +} + +/** + * The speaker id a model should use for a language when the caller's + * speaker was chosen for another language. `undefined` keeps the caller's + * speaker. + * @param {string} modelId + * @param {string} language + * @returns {number | undefined} + */ +export function getLocalTtsDefaultSpeaker(modelId, language) { + const speaker = LOCAL_TTS_MODEL_CATALOG[modelId]?.defaultSpeakerByLanguage?.[language]; + return Number.isInteger(speaker) ? speaker : undefined; +} + /** * @param {string} modelsDir * @param {string} modelId diff --git a/packages/web/server/lib/dictation/local/model-catalog.test.js b/packages/web/server/lib/dictation/local/model-catalog.test.js new file mode 100644 index 00000000..e14a66aa --- /dev/null +++ b/packages/web/server/lib/dictation/local/model-catalog.test.js @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_LOCAL_TTS_MODEL, + LOCAL_TTS_MODEL_CATALOG, + getLocalSttModelSpec, + getLocalTtsDefaultSpeaker, + resolveLocalTtsModelForLanguage, +} from './model-catalog.js'; + +describe('local TTS catalog', () => { + it('keeps the selected model when it speaks the language', () => { + expect(resolveLocalTtsModelForLanguage('en', DEFAULT_LOCAL_TTS_MODEL)).toBe(DEFAULT_LOCAL_TTS_MODEL); + expect(resolveLocalTtsModelForLanguage('zh', 'kokoro-multi-lang-v1_1')).toBe('kokoro-multi-lang-v1_1'); + }); + + it('picks a catalog model for a language the selected model lacks', () => { + expect(resolveLocalTtsModelForLanguage('uk', DEFAULT_LOCAL_TTS_MODEL)).toBe('piper-uk_UA-lada-x_low'); + expect(resolveLocalTtsModelForLanguage('zh', DEFAULT_LOCAL_TTS_MODEL)).toBe('kokoro-multi-lang-v1_1'); + }); + + it('returns null for a language no model covers', () => { + expect(resolveLocalTtsModelForLanguage('xx', DEFAULT_LOCAL_TTS_MODEL)).toBeNull(); + }); + + it('gives Chinese a Chinese speaker on the multi-language Kokoro', () => { + expect(getLocalTtsDefaultSpeaker('kokoro-multi-lang-v1_1', 'zh')).toBe(3); + expect(getLocalTtsDefaultSpeaker('kokoro-multi-lang-v1_1', 'en')).toBe(0); + expect(getLocalTtsDefaultSpeaker('piper-uk_UA-lada-x_low', 'uk')).toBeUndefined(); + }); + + it('every TTS entry declares its languages and installable files', () => { + for (const [id, spec] of Object.entries(LOCAL_TTS_MODEL_CATALOG)) { + expect(spec.languages.length, id).toBeGreaterThan(0); + expect(spec.archiveUrl, id).toMatch(/^https:\/\/github\.com\/k2-fsa\/sherpa-onnx\/releases\/download\/tts-models\//); + const resolved = getLocalSttModelSpec(id); + expect(resolved.requiredFiles, id).toContain(spec.files.model); + for (const key of spec.lexicon ?? []) { + expect(spec.files[key], `${id} lexicon ${key}`).toBeTruthy(); + } + } + }); +}); diff --git a/packages/web/server/lib/dictation/local/sherpa-tts.js b/packages/web/server/lib/dictation/local/sherpa-tts.js index f4fa7972..589bae69 100644 --- a/packages/web/server/lib/dictation/local/sherpa-tts.js +++ b/packages/web/server/lib/dictation/local/sherpa-tts.js @@ -1,5 +1,5 @@ /** - * Sherpa-onnx offline TTS (Kokoro). Runs inside the dictation worker process + * Sherpa-onnx offline TTS (Kokoro and Piper/VITS). Runs inside the dictation worker process * only — never load the native addon in the main server process. */ @@ -23,20 +23,49 @@ function float32ToPcm16le(samples) { return Buffer.from(out.buffer, out.byteOffset, out.byteLength); } +/** + * sherpa-onnx model config for one catalog entry. Kokoro carries a voices + * bank (speaker ids) and optional lexicons; a Piper/VITS model is a single + * voice with espeak-ng phonemization. + * @param {{ modelDir: string, type?: string, files: Record, lexicon?: string[] }} config + */ +function buildModelConfig(config) { + const file = (key, label) => { + const filePath = path.join(config.modelDir, config.files[key]); + assertFileExists(filePath, label); + return filePath; + }; + const modelPath = file('model', 'TTS model'); + const tokensPath = file('tokens', 'TTS tokens'); + + if (config.type === 'vits') { + // Piper models phonemize through espeak-ng (`espeakData`); character + // models (Coqui) read the text directly and carry no espeak data. + const dataDir = config.files.espeakData ? file('espeakData', 'TTS espeak-ng dataDir') : ''; + return { vits: { model: modelPath, tokens: tokensPath, ...(dataDir ? { dataDir } : {}), lengthScale: 1.0 } }; + } + + const dataDir = file('espeakData', 'TTS espeak-ng dataDir'); + const voicesPath = file('voices', 'TTS voices'); + const lexicon = (config.lexicon ?? []).map((key) => file(key, 'TTS lexicon')).join(','); + return { + kokoro: { + model: modelPath, + voices: voicesPath, + tokens: tokensPath, + dataDir, + lengthScale: 1.0, + ...(lexicon ? { lexicon } : {}), + }, + }; +} + export class SherpaTtsEngine { /** - * @param {{ modelDir: string, files: { model: string, voices: string, tokens: string, espeakData: string }, numThreads?: number }} config + * @param {{ modelDir: string, type?: string, files: Record, lexicon?: string[], numThreads?: number }} config */ constructor(config) { - const modelPath = path.join(config.modelDir, config.files.model); - const voicesPath = path.join(config.modelDir, config.files.voices); - const tokensPath = path.join(config.modelDir, config.files.tokens); - const dataDir = path.join(config.modelDir, config.files.espeakData); - - assertFileExists(modelPath, 'TTS model'); - assertFileExists(voicesPath, 'TTS voices'); - assertFileExists(tokensPath, 'TTS tokens'); - assertFileExists(dataDir, 'TTS espeak-ng dataDir'); + const model = buildModelConfig(config); const sherpa = loadSherpaOnnxNode(); if (typeof sherpa.OfflineTts !== 'function') { @@ -44,15 +73,7 @@ export class SherpaTtsEngine { } this.tts = new sherpa.OfflineTts({ - model: { - kokoro: { - model: modelPath, - voices: voicesPath, - tokens: tokensPath, - dataDir, - lengthScale: 1.0, - }, - }, + model, numThreads: config.numThreads ?? 2, provider: 'cpu', maxNumSentences: 1, diff --git a/packages/web/server/lib/dictation/local/worker-process.js b/packages/web/server/lib/dictation/local/worker-process.js index 3a5c34b8..06a7ac8b 100644 --- a/packages/web/server/lib/dictation/local/worker-process.js +++ b/packages/web/server/lib/dictation/local/worker-process.js @@ -102,7 +102,9 @@ function getTtsEngine(modelsDir, modelId) { const spec = getLocalSttModelSpec(modelId); const created = new SherpaTtsEngine({ modelDir: getLocalSttModelDir(modelsDir, modelId), + type: spec.type, files: spec.files, + lexicon: spec.lexicon, numThreads: 2, }); ttsEngines.set(key, created); diff --git a/packages/web/server/lib/dictation/runtime.js b/packages/web/server/lib/dictation/runtime.js index 8fae1fcf..59437a12 100644 --- a/packages/web/server/lib/dictation/runtime.js +++ b/packages/web/server/lib/dictation/runtime.js @@ -63,6 +63,8 @@ export function createDictationRuntime({ model: typeof req.body?.model === 'string' ? req.body.model : undefined, speakerId: Number.isInteger(req.body?.speakerId) ? req.body.speakerId : undefined, speed: typeof req.body?.speed === 'number' ? req.body.speed : undefined, + language: req.body?.language === 'auto' ? 'auto' : undefined, + languageSample: typeof req.body?.languageSample === 'string' ? req.body.languageSample.slice(0, 4000) : undefined, }); if (result.error) { res.status(503).json({ @@ -73,6 +75,8 @@ export function createDictationRuntime({ return; } res.setHeader('Content-Type', result.format || 'audio/wav'); + res.setHeader('X-Speech-Model', result.modelId); + if (result.language) res.setHeader('X-Speech-Language', result.language); res.send(result.audio); } catch (error) { res.status(500).json({ error: error?.message || 'Failed to synthesize speech' }); diff --git a/packages/web/server/lib/dictation/service.js b/packages/web/server/lib/dictation/service.js index 7dc555d2..e40bc253 100644 --- a/packages/web/server/lib/dictation/service.js +++ b/packages/web/server/lib/dictation/service.js @@ -1,3 +1,4 @@ +import { detectTextLanguage } from '../tts/language-detect.js'; /** * Dictation service: resolves STT providers, tracks local model download * state, and exposes a readiness snapshot for the status route. @@ -16,6 +17,8 @@ import { OpenAICompatibleTranscriptionSession } from './openai-compatible-sessio import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, + getLocalTtsDefaultSpeaker, + resolveLocalTtsModelForLanguage, LOCAL_STT_MODEL_CATALOG, LOCAL_STT_MODEL_IDS, LOCAL_TTS_MODEL_CATALOG, @@ -220,10 +223,30 @@ export function createDictationService({ modelsDir }) { /** * Synthesize speech with the local TTS model. Returns WAV bytes, or a * readiness error while the model is missing/downloading. - * @param {{ text: string, model?: string, speakerId?: number, speed?: number }} options + * + * With `language: 'auto'` the text's language decides the model: the + * caller's model when it speaks that language, otherwise the catalog + * model for it (downloaded on first use, reported as in-progress until it + * lands). The caller's speaker id is kept only on the caller's model; a + * substitute model starts from its own default speaker for the language. + * A language no catalog model covers keeps the caller's model, so text is + * never silently dropped. + * `languageSample` is the whole message the chunk belongs to (or a prefix + * of it): the language is judged on that, never on a short chunk alone. + * @param {{ text: string, model?: string, speakerId?: number, speed?: number, language?: string, languageSample?: string }} options */ - const synthesizeSpeech = async ({ text, model, speakerId, speed }) => { - const modelId = isLocalTtsModelId(model) ? model : DEFAULT_LOCAL_TTS_MODEL; + const synthesizeSpeech = async ({ text, model, speakerId, speed, language, languageSample }) => { + const requestedModelId = isLocalTtsModelId(model) ? model : DEFAULT_LOCAL_TTS_MODEL; + let modelId = requestedModelId; + let resolvedLanguage = null; + if (language === 'auto') { + resolvedLanguage = detectTextLanguage(languageSample || text).language; + const forLanguage = resolveLocalTtsModelForLanguage(resolvedLanguage, requestedModelId); + if (forLanguage && forLanguage !== requestedModelId) { + modelId = forLanguage; + speakerId = getLocalTtsDefaultSpeaker(modelId, resolvedLanguage); + } + } const installed = await isLocalSttModelInstalled(modelsDir, modelId); if (!installed) { const state = downloadStates.get(modelId); @@ -251,7 +274,7 @@ export function createDictationService({ modelsDir }) { speakerId, speed, }); - return { audio: result.audio, format: result.format }; + return { audio: result.audio, format: result.format, modelId, language: resolvedLanguage }; }; /** diff --git a/packages/web/server/lib/tts/DOCUMENTATION.md b/packages/web/server/lib/tts/DOCUMENTATION.md index 81a46760..2e0c4352 100644 --- a/packages/web/server/lib/tts/DOCUMENTATION.md +++ b/packages/web/server/lib/tts/DOCUMENTATION.md @@ -11,6 +11,7 @@ This module provides server-side Text-to-Speech services using OpenAI's TTS API. - `packages/web/server/lib/text/summarization.js`: Shared text summarization stub and sanitization utilities. It performs no external Zen calls. - `packages/web/server/lib/tts/stt.js`: STT proxy for OpenAI-compatible transcription endpoints. - `packages/web/server/lib/tts/base-url.js`: shared base URL validation and normalization for custom OpenAI-compatible endpoints. +- `packages/web/server/lib/tts/language-detect.js`: dependency-free language detection for voice selection (`detectTextLanguage`, `pickVoiceForLanguage`, `languageOfLocale`). Used by the macOS `say` route (`language: 'auto'` switches to an installed voice whose locale matches the text; the response carries `X-Speech-Voice` and `X-Speech-Language`) and by the dictation module's local TTS model choice. ## Public exports diff --git a/packages/web/server/lib/tts/language-detect.js b/packages/web/server/lib/tts/language-detect.js new file mode 100644 index 00000000..d87968e6 --- /dev/null +++ b/packages/web/server/lib/tts/language-detect.js @@ -0,0 +1,210 @@ +/** + * Language detection for text-to-speech voice selection. + * + * Picks the language a piece of chat text is written in so a TTS provider + * can choose a matching voice or model. Deliberately small and dependency + * free: the writing system decides most cases outright, and Latin-script + * languages are told apart by function words and characteristic letters. + * The answer is a best effort for voice selection, not a linguistic claim — + * an unknown language falls back to English rather than failing. + */ + +const SCRIPT_RANGES = [ + ['hangul', /[가-힯ᄀ-ᇿ㄰-㆏]/g], + ['kana', /[぀-ヿ]/g], + ['han', /[一-鿿㐀-䶿]/g], + ['cyrillic', /[Ѐ-ӿ]/g], + ['greek', /[Ͱ-Ͽ]/g], + ['arabic', /[؀-ۿ]/g], + ['hebrew', /[֐-׿]/g], + ['thai', /[฀-๿]/g], + ['devanagari', /[ऀ-ॿ]/g], + ['latin', /[A-Za-zÀ-ɏ]/g], +]; + +const SCRIPT_LANGUAGE = { + hangul: 'ko', + greek: 'el', + arabic: 'ar', + hebrew: 'he', + thai: 'th', + devanagari: 'hi', +}; + +// Letters that only (or overwhelmingly) occur in one language of a script. +const LATIN_MARKERS = { + pl: /[łęąńśźż]/i, + cs: /[řěůťďň]/i, + tr: /[ğışİ]/, + pt: /[ãõ]/i, + es: /[ñ¿¡]/, + de: /[ß]/, + fr: /[œ]/i, + sv: /[å]/i, +}; + +// Frequent function words per language. Scored by whole-word hits; every +// list has the same length so scores stay comparable. +const STOPWORDS = { + en: ['the', 'and', 'is', 'to', 'of', 'that', 'you', 'with', 'for', 'this', 'are', 'it', 'not', 'have', 'can', 'will', 'your', 'from', 'which', 'when'], + de: ['und', 'der', 'die', 'das', 'ist', 'nicht', 'mit', 'ein', 'eine', 'auch', 'sich', 'auf', 'für', 'wird', 'werden', 'oder', 'aber', 'wenn', 'sind', 'kann'], + fr: ['le', 'la', 'les', 'et', 'est', 'une', 'des', 'pour', 'que', 'qui', 'dans', 'pas', 'vous', 'sur', 'avec', 'sont', 'nous', 'cette', 'mais', 'plus'], + es: ['el', 'la', 'los', 'las', 'que', 'es', 'una', 'por', 'para', 'con', 'del', 'como', 'pero', 'más', 'este', 'esta', 'son', 'tiene', 'puede', 'también'], + it: ['il', 'la', 'che', 'di', 'è', 'una', 'per', 'non', 'con', 'del', 'della', 'come', 'sono', 'anche', 'questo', 'questa', 'gli', 'nel', 'più', 'essere'], + pt: ['o', 'a', 'os', 'as', 'que', 'é', 'uma', 'para', 'com', 'não', 'do', 'da', 'como', 'mas', 'também', 'este', 'esta', 'são', 'você', 'pode'], + pl: ['i', 'nie', 'jest', 'się', 'na', 'to', 'że', 'jak', 'ale', 'dla', 'oraz', 'przez', 'czy', 'tym', 'jego', 'można', 'jeśli', 'tego', 'które', 'także'], + nl: ['de', 'het', 'een', 'en', 'van', 'is', 'niet', 'dat', 'met', 'voor', 'ook', 'zijn', 'maar', 'als', 'wordt', 'deze', 'kan', 'naar', 'bij', 'dan'], + cs: ['a', 'je', 'se', 'na', 'to', 'že', 'jak', 'ale', 'pro', 'nebo', 'jsou', 'může', 'také', 'tento', 'když', 'jeho', 'které', 'být', 'aby', 'ještě'], + tr: ['ve', 'bir', 'bu', 'için', 'ile', 'de', 'da', 'ama', 'gibi', 'daha', 'var', 'olarak', 'çok', 'ne', 'her', 'kadar', 'sonra', 'değil', 'olan', 'ise'], + sv: ['och', 'att', 'det', 'är', 'en', 'som', 'för', 'inte', 'med', 'till', 'den', 'kan', 'har', 'ett', 'men', 'också', 'eller', 'från', 'när', 'vara'], + uk: ['і', 'та', 'що', 'це', 'не', 'як', 'для', 'він', 'вона', 'але', 'або', 'також', 'тільки', 'вже', 'якщо', 'його', 'цей', 'ця', 'бути', 'коли'], + ru: ['и', 'что', 'это', 'не', 'как', 'для', 'он', 'она', 'но', 'или', 'также', 'только', 'уже', 'если', 'его', 'этот', 'эта', 'быть', 'когда', 'чтобы'], +}; + +const LATIN_LANGUAGES = ['en', 'de', 'fr', 'es', 'it', 'pt', 'pl', 'nl', 'cs', 'tr', 'sv']; +const CYRILLIC_LANGUAGES = ['uk', 'ru']; + +const countMatches = (text, pattern) => { + const matches = text.match(pattern); + return matches ? matches.length : 0; +}; + +const scoreStopwords = (words, languages) => { + const scores = {}; + for (const language of languages) { + const list = new Set(STOPWORDS[language]); + let hits = 0; + for (const word of words) { + if (list.has(word)) hits += 1; + } + scores[language] = hits; + } + return scores; +}; + +const bestOf = (scores, fallback) => { + let best = fallback; + let bestScore = 0; + for (const [language, score] of Object.entries(scores)) { + if (score > bestScore) { + best = language; + bestScore = score; + } + } + return best; +}; + +const pickByMarkers = (text, markers) => { + for (const [language, pattern] of Object.entries(markers)) { + if (pattern.test(text)) return language; + } + return null; +}; + +/** + * @param {string} text + * @returns {{ language: string, script: string }} BCP-47 primary language subtag and the dominant script. + */ +export function detectTextLanguage(text) { + const source = typeof text === 'string' ? text : ''; + const counts = SCRIPT_RANGES.map(([script, pattern]) => [script, countMatches(source, pattern)]); + const letters = counts.reduce((sum, [, count]) => sum + count, 0); + if (letters === 0) return { language: 'en', script: 'latin' }; + + // Kana settles Japanese even when Han dominates the character count. + const kana = counts.find(([script]) => script === 'kana')?.[1] ?? 0; + const han = counts.find(([script]) => script === 'han')?.[1] ?? 0; + if (kana > 0 && kana + han >= letters * 0.3) return { language: 'ja', script: 'kana' }; + if (han > 0 && han >= letters * 0.3) return { language: 'zh', script: 'han' }; + + const [script] = counts.reduce((best, entry) => (entry[1] > best[1] ? entry : best)); + + if (script in SCRIPT_LANGUAGE) return { language: SCRIPT_LANGUAGE[script], script }; + + const words = source.toLowerCase().split(/[^\p{L}\p{M}']+/u).filter(Boolean); + + if (script === 'cyrillic') { + const scores = scoreStopwords(words, CYRILLIC_LANGUAGES); + const ukMarkers = countMatches(source, /[іїєґ]/gi); + const ruMarkers = countMatches(source, /[ыэъё]/gi); + // Letters decide: the two alphabets differ in letters that occur in + // nearly every sentence. Function words only settle a text that shows + // neither set, and a text with no Russian-only letters is far more + // likely Ukrainian than the reverse, so that tie goes to Ukrainian. + if (ukMarkers !== ruMarkers) return { language: ukMarkers > ruMarkers ? 'uk' : 'ru', script }; + if (scores.uk !== scores.ru) return { language: scores.uk > scores.ru ? 'uk' : 'ru', script }; + return { language: ruMarkers > 0 ? 'ru' : 'uk', script }; + } + + const scores = scoreStopwords(words, LATIN_LANGUAGES); + const marked = pickByMarkers(source, LATIN_MARKERS); + // A characteristic letter outranks stopword counts unless another language + // clearly dominates the function words (a German text quoting "façade"). + if (marked && scores[marked] * 2 >= scores[bestOf(scores, marked)]) { + return { language: marked, script }; + } + return { language: bestOf(scores, 'en'), script }; +} + +/** + * Map a detected language onto the locales a voice list uses (`uk_UA`, + * `en_US`...). Returns the preferred locale prefixes in order. + * @param {string} language + * @returns {string[]} + */ +function localePrefixesForLanguage(language) { + const table = { + en: ['en_US', 'en_GB', 'en'], + uk: ['uk_UA', 'uk'], + ru: ['ru_RU', 'ru'], + de: ['de_DE', 'de'], + fr: ['fr_FR', 'fr_CA', 'fr'], + es: ['es_ES', 'es_MX', 'es'], + it: ['it_IT', 'it'], + pt: ['pt_BR', 'pt_PT', 'pt'], + pl: ['pl_PL', 'pl'], + nl: ['nl_NL', 'nl_BE', 'nl'], + cs: ['cs_CZ', 'cs'], + tr: ['tr_TR', 'tr'], + sv: ['sv_SE', 'sv'], + zh: ['zh_CN', 'zh_TW', 'zh_HK', 'zh'], + ja: ['ja_JP', 'ja'], + ko: ['ko_KR', 'ko'], + el: ['el_GR', 'el'], + ar: ['ar_001', 'ar_SA', 'ar'], + he: ['he_IL', 'he'], + th: ['th_TH', 'th'], + hi: ['hi_IN', 'hi'], + }; + return table[language] ?? [language]; +} + +/** + * Choose a voice for a language from a `say`-style voice list. + * Prefers an enhanced/premium variant of a matching voice, then any voice of + * the exact locale, then any voice of the language. Returns null when the + * list has no voice for that language. + * @param {string} language + * @param {ReadonlyArray<{ name: string, locale: string }>} voices + * @returns {string | null} + */ +export function pickVoiceForLanguage(language, voices) { + const prefixes = localePrefixesForLanguage(language); + for (const prefix of prefixes) { + const matching = voices.filter((voice) => voice.locale === prefix || voice.locale.startsWith(`${prefix}_`) || (prefix === language && voice.locale.startsWith(`${language}_`))); + if (matching.length === 0) continue; + const enhanced = matching.find((voice) => /\((Enhanced|Premium)\)/i.test(voice.name)); + return (enhanced ?? matching[0]).name; + } + return null; +} + +/** + * Language of a voice, from its locale (`uk_UA` → `uk`). + * @param {string | null | undefined} locale + * @returns {string | null} + */ +export function languageOfLocale(locale) { + if (typeof locale !== 'string' || !locale) return null; + return locale.split(/[_-]/)[0].toLowerCase(); +} diff --git a/packages/web/server/lib/tts/language-detect.test.js b/packages/web/server/lib/tts/language-detect.test.js new file mode 100644 index 00000000..a8c0f2c3 --- /dev/null +++ b/packages/web/server/lib/tts/language-detect.test.js @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { detectTextLanguage, languageOfLocale, pickVoiceForLanguage } from './language-detect.js'; + +describe('detectTextLanguage', () => { + it.each([ + ['en', 'The build is green and the tests pass, so you can merge this now.'], + ['uk', 'Привіт! Це тестове повідомлення, і воно написане українською мовою.'], + ['ru', 'Привет! Это тестовое сообщение, и оно написано на русском языке.'], + ['de', 'Die Änderung ist fertig und die Tests laufen ohne Fehler durch.'], + ['fr', 'La modification est prête et les tests passent sans erreur.'], + ['es', 'El cambio está listo y las pruebas pasan sin errores.'], + ['it', 'La modifica è pronta e i test passano senza errori.'], + ['pt', 'A alteração está pronta e os testes passam sem erros, você pode continuar.'], + ['pl', 'Zmiana jest gotowa i testy przechodzą bez błędów.'], + ['nl', 'De wijziging is klaar en de tests slagen zonder fouten.'], + ['cs', 'Změna je hotová a testy procházejí bez chyb.'], + ['tr', 'Değişiklik hazır ve testler hatasız geçiyor.'], + ['sv', 'Ändringen är klar och testerna går igenom utan fel.'], + ['zh', '修改已经完成,所有测试都通过了。'], + ['ja', '変更が完了し、すべてのテストに合格しました。'], + ['ko', '변경이 완료되었고 모든 테스트를 통과했습니다.'], + ])('detects %s', (language, text) => { + expect(detectTextLanguage(text).language).toBe(language); + }); + + it.each([ + ['uk', 'Готово. Запушено.'], + ['uk', 'Все ок'], + ['uk', 'Добре, давай так зробимо'], + ['ru', 'Хорошо, давай так и сделаем'], + ['ru', 'Готово, всё запушено.'], + ])('tells short %s phrases apart by letters', (language, text) => { + expect(detectTextLanguage(text).language).toBe(language); + }); + + it('falls back to English for text without letters', () => { + expect(detectTextLanguage('1234 ... !!!').language).toBe('en'); + expect(detectTextLanguage('').language).toBe('en'); + }); + + it('does not let a single quoted foreign word flip an English paragraph', () => { + const text = 'The façade of the building is the part that you see from the street, and it is not the same as the interior.'; + expect(detectTextLanguage(text).language).toBe('en'); + }); +}); + +describe('pickVoiceForLanguage', () => { + const voices = [ + { name: 'Samantha', locale: 'en_US' }, + { name: 'Daniel', locale: 'en_GB' }, + { name: 'Lesya', locale: 'uk_UA' }, + { name: 'Lesya (Enhanced)', locale: 'uk_UA' }, + { name: 'Milena', locale: 'ru_RU' }, + { name: 'Anna', locale: 'de_DE' }, + ]; + + it('prefers the enhanced variant of a matching voice', () => { + expect(pickVoiceForLanguage('uk', voices)).toBe('Lesya (Enhanced)'); + }); + + it('prefers the primary locale of a language', () => { + expect(pickVoiceForLanguage('en', voices)).toBe('Samantha'); + }); + + it('returns null when no voice speaks the language', () => { + expect(pickVoiceForLanguage('ja', voices)).toBeNull(); + }); +}); + +describe('languageOfLocale', () => { + it('reads the language subtag', () => { + expect(languageOfLocale('uk_UA')).toBe('uk'); + expect(languageOfLocale('en-GB')).toBe('en'); + expect(languageOfLocale(null)).toBeNull(); + }); +}); diff --git a/packages/web/server/lib/tts/routes.js b/packages/web/server/lib/tts/routes.js index 5d2c4618..2f2074ff 100644 --- a/packages/web/server/lib/tts/routes.js +++ b/packages/web/server/lib/tts/routes.js @@ -2,6 +2,8 @@ import express from 'express'; import { normalizeCustomOpenAIBaseURL } from './base-url.js'; import { summarizeText, sanitizeForTTS, sanitizeForNote } from '../text/summarization.js'; +import { detectTextLanguage, languageOfLocale, pickVoiceForLanguage } from './language-detect.js'; + export function registerTtsRoutes(app, { sayTTSCapability }) { let ttsModulePromise = null; const getTtsModule = async () => { @@ -154,7 +156,8 @@ export function registerTtsRoutes(app, { sayTTSCapability }) { // macOS 'say' command TTS speak endpoint app.post('/api/tts/say/speak', async (req, res) => { try { - const { text, voice = 'Samantha', rate = 200 } = req.body || {}; + const { text, rate = 200, language, languageSample } = req.body || {}; + let voice = typeof req.body?.voice === 'string' && req.body.voice.trim() ? req.body.voice.trim() : 'Samantha'; if (!text || typeof text !== 'string' || !text.trim()) { return res.status(400).json({ error: 'Text is required' }); @@ -164,6 +167,23 @@ export function registerTtsRoutes(app, { sayTTSCapability }) { if (process.platform !== 'darwin') { return res.status(503).json({ error: 'macOS say command not available on this platform' }); } + + // `language: 'auto'`: keep the chosen voice while it speaks the text's + // language, otherwise switch to an installed voice that does. A + // language with no installed voice keeps the chosen voice — say still + // reads the text, just with an accent — rather than failing. + let resolvedLanguage = null; + if (language === 'auto') { + const capability = await sayTTSCapability; + const voices = Array.isArray(capability?.voices) ? capability.voices : []; + const sample = typeof languageSample === 'string' && languageSample.trim() ? languageSample.slice(0, 4000) : text; + resolvedLanguage = detectTextLanguage(sample).language; + const chosen = voices.find((entry) => entry.name === voice); + if (languageOfLocale(chosen?.locale) !== resolvedLanguage) { + const match = pickVoiceForLanguage(resolvedLanguage, voices); + if (match) voice = match; + } + } const { exec } = await import('child_process'); const { promisify } = await import('util'); @@ -195,6 +215,8 @@ export function registerTtsRoutes(app, { sayTTSCapability }) { // Send audio response res.setHeader('Content-Type', 'audio/mp4'); + res.setHeader('X-Speech-Voice', voice); + if (resolvedLanguage) res.setHeader('X-Speech-Language', resolvedLanguage); res.setHeader('Content-Length', audioBuffer.length); res.send(audioBuffer); diff --git a/packages/web/server/lib/tts/routes.test.js b/packages/web/server/lib/tts/routes.test.js index f4940265..fce960a4 100644 --- a/packages/web/server/lib/tts/routes.test.js +++ b/packages/web/server/lib/tts/routes.test.js @@ -33,6 +33,32 @@ describe('tts routes', () => { }); }); + it('switches the say voice to the language of the text when asked to', async () => { + const capability = Promise.resolve({ + available: true, + voices: [ + { name: 'Samantha', locale: 'en_US' }, + { name: 'Lesya', locale: 'uk_UA' }, + { name: 'Lesya (Enhanced)', locale: 'uk_UA' }, + ], + }); + const app = createApp(capability); + const response = await request(app) + .post('/api/tts/say/speak') + .send({ text: 'Привіт! Це відповідь українською мовою, і вона досить довга.', voice: 'Samantha', language: 'auto' }); + + // On macOS the route synthesizes; elsewhere it refuses before running say. + // Either way the chosen voice must be the Ukrainian one when the platform + // allows the request to proceed. + if (process.platform === 'darwin') { + expect(response.status).toBe(200); + expect(response.headers['x-speech-voice']).toBe('Lesya (Enhanced)'); + expect(response.headers['x-speech-language']).toBe('uk'); + } else { + expect(response.status).toBe(503); + } + }); + it('returns local note fallback while model summarization is retired', async () => { const response = await request(createApp()) .post('/api/text/summarize')