feat(voice): match local and macOS voices to the language of the text
Text-to-speech picked one voice regardless of what language a reply was in. A dependency-free language detector (script, marker letters, function words) now decides the language of the whole message once; with the new "Match the voice to the language of the text" setting the local provider switches to a catalog model for that language (Kokoro zh/en and Piper models for 12 languages, downloaded on first use like the existing model) and macOS say switches to an installed voice whose locale matches. The local voice picker lists voices of every installed model, and the settings show which language models are on disk. The Ukrainian Piper medium build is a character-level model that sherpa-onnx turns into noise, so the espeak-based Lada build is used instead. Claude-Session: https://claude.ai/code/session_017TK5JAYDfT3Fotc23UEg98
This commit is contained in:
Bohdan Triapitsyn
@@ -71,6 +71,7 @@ const LOCAL_STT_MODELS = [
|
||||
|
||||
interface DictationModelState {
|
||||
id: string;
|
||||
description?: string;
|
||||
installed: boolean;
|
||||
downloading: boolean;
|
||||
downloadProgress: number | null;
|
||||
@@ -288,10 +289,32 @@ const KOKORO_VOICE_OPTIONS = [
|
||||
|
||||
const LOCAL_TTS_MODEL_ID = 'kokoro-en-v0_19';
|
||||
|
||||
const LocalTtsModelStatus = () => {
|
||||
const { t } = useI18n();
|
||||
const [model, setModel] = useState<DictationModelState | null>(null);
|
||||
const [requesting, setRequesting] = useState(false);
|
||||
const KOKORO_MULTI_LANG_MODEL_ID = 'kokoro-multi-lang-v1_1';
|
||||
// A few named speakers out of the 103 in the Chinese/English Kokoro build.
|
||||
const KOKORO_MULTI_LANG_VOICE_OPTIONS = [
|
||||
{ id: 0, label: 'Maple (af)' },
|
||||
{ id: 1, label: 'Sol (af)' },
|
||||
{ id: 2, label: 'Vale (bf)' },
|
||||
{ id: 3, label: 'Xiaoxiao (zf)' },
|
||||
{ id: 58, label: 'Yunxi (zm)' },
|
||||
];
|
||||
|
||||
interface LocalTtsVoiceOption {
|
||||
modelId: string;
|
||||
speakerId: number;
|
||||
label: string;
|
||||
}
|
||||
|
||||
const localTtsVoiceKey = (modelId: string, speakerId: number): string => `${modelId}:${speakerId}`;
|
||||
|
||||
/**
|
||||
* Local TTS models as the server reports them, plus the actions Settings
|
||||
* offers on them. Shared by the model list and the voice picker so both see
|
||||
* the same install state.
|
||||
*/
|
||||
const useLocalTtsModels = () => {
|
||||
const [models, setModels] = useState<DictationModelState[]>([]);
|
||||
const [requestingId, setRequestingId] = useState<string | null>(null);
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
try {
|
||||
@@ -300,11 +323,8 @@ const LocalTtsModelStatus = () => {
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
const entry = Array.isArray(data?.ttsModels)
|
||||
? data.ttsModels.find((m: DictationModelState) => m.id === LOCAL_TTS_MODEL_ID)
|
||||
: null;
|
||||
if (entry) {
|
||||
setModel(entry);
|
||||
if (Array.isArray(data?.ttsModels)) {
|
||||
setModels(data.ttsModels);
|
||||
}
|
||||
} catch {
|
||||
// Display-only status; keep the previous state on fetch failure.
|
||||
@@ -315,81 +335,118 @@ const LocalTtsModelStatus = () => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
const anyDownloading = models.some((model) => model.downloading);
|
||||
useEffect(() => {
|
||||
if (!model?.downloading) {
|
||||
if (!anyDownloading) {
|
||||
return;
|
||||
}
|
||||
const interval = setInterval(() => {
|
||||
void refresh();
|
||||
}, 2000);
|
||||
return () => clearInterval(interval);
|
||||
}, [model?.downloading, refresh]);
|
||||
}, [anyDownloading, refresh]);
|
||||
|
||||
const request = async (method: 'POST' | 'DELETE') => {
|
||||
setRequesting(true);
|
||||
const request = useCallback(async (modelId: string, method: 'POST' | 'DELETE') => {
|
||||
setRequestingId(modelId);
|
||||
try {
|
||||
const path = method === 'POST'
|
||||
? `/api/dictation/models/${LOCAL_TTS_MODEL_ID}/download`
|
||||
: `/api/dictation/models/${LOCAL_TTS_MODEL_ID}`;
|
||||
? `/api/dictation/models/${modelId}/download`
|
||||
: `/api/dictation/models/${modelId}`;
|
||||
await runtimeFetch(path, { method });
|
||||
await refresh();
|
||||
} catch {
|
||||
// Status refresh reports errors.
|
||||
} finally {
|
||||
setRequesting(false);
|
||||
setRequestingId(null);
|
||||
}
|
||||
};
|
||||
}, [refresh]);
|
||||
|
||||
if (!model) {
|
||||
return { models, requestingId, request, refresh };
|
||||
};
|
||||
|
||||
// Voices the picker offers: Kokoro speakers for the Kokoro models, one voice
|
||||
// per installed Piper model. Only installed models (plus the default) appear,
|
||||
// so a language model the server fetched on its own becomes selectable once
|
||||
// it is on disk.
|
||||
const buildLocalTtsVoiceOptions = (models: DictationModelState[]): LocalTtsVoiceOption[] => {
|
||||
const options: LocalTtsVoiceOption[] = KOKORO_VOICE_OPTIONS.map((voice) => ({
|
||||
modelId: LOCAL_TTS_MODEL_ID,
|
||||
speakerId: voice.id,
|
||||
label: voice.label,
|
||||
}));
|
||||
for (const model of models) {
|
||||
if (model.id === LOCAL_TTS_MODEL_ID || !model.installed) continue;
|
||||
if (model.id === KOKORO_MULTI_LANG_MODEL_ID) {
|
||||
for (const voice of KOKORO_MULTI_LANG_VOICE_OPTIONS) {
|
||||
options.push({ modelId: model.id, speakerId: voice.id, label: `${voice.label} · Kokoro zh/en` });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
options.push({ modelId: model.id, speakerId: 0, label: model.description ?? model.id });
|
||||
}
|
||||
return options;
|
||||
};
|
||||
|
||||
const LocalTtsModelStatus = ({ models, requestingId, request }: ReturnType<typeof useLocalTtsModels>) => {
|
||||
const { t } = useI18n();
|
||||
|
||||
// The default English model is always listed; language models the server
|
||||
// fetched on its own appear once they are installed or downloading, so
|
||||
// the list shows what is on disk rather than the whole catalog.
|
||||
const visible = models.filter((model) => model.id === LOCAL_TTS_MODEL_ID || model.installed || model.downloading);
|
||||
if (visible.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 py-1.5">
|
||||
<span className="typography-ui-label text-foreground">Kokoro</span>
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">305 MB</span>
|
||||
{model.installed ? (
|
||||
<>
|
||||
<Icon
|
||||
name="checkbox-circle"
|
||||
className="h-4 w-4 text-[var(--status-success)]"
|
||||
aria-label={t('settings.voice.page.stt.modelInstalled')}
|
||||
/>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
|
||||
disabled={requesting}
|
||||
onClick={() => { void request('DELETE'); }}
|
||||
title={t('settings.voice.page.stt.modelDelete')}
|
||||
aria-label={t('settings.voice.page.stt.modelDelete')}
|
||||
>
|
||||
<Icon name="delete-bin" className="h-4 w-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : model.downloading ? (
|
||||
<span className="flex items-center gap-1.5">
|
||||
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">
|
||||
{typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''}
|
||||
</span>
|
||||
</span>
|
||||
) : (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0"
|
||||
disabled={requesting}
|
||||
onClick={() => { void request('POST'); }}
|
||||
title={t('settings.voice.page.stt.modelDownload')}
|
||||
aria-label={t('settings.voice.page.stt.modelDownload')}
|
||||
>
|
||||
<Icon name="download" className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
{model.downloadError ? (
|
||||
<span className="typography-meta text-[var(--status-error)]">{model.downloadError}</span>
|
||||
) : null}
|
||||
<div className="flex flex-col">
|
||||
{visible.map((model) => (
|
||||
<div key={model.id} className="flex items-center gap-2 py-1.5">
|
||||
<span className="typography-ui-label text-foreground">{model.description ?? model.id}</span>
|
||||
{model.installed ? (
|
||||
<>
|
||||
<Icon
|
||||
name="checkbox-circle"
|
||||
className="h-4 w-4 text-[var(--status-success)]"
|
||||
aria-label={t('settings.voice.page.stt.modelInstalled')}
|
||||
/>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0 text-muted-foreground hover:text-[var(--status-error)]"
|
||||
disabled={requestingId !== null}
|
||||
onClick={() => { void request(model.id, 'DELETE'); }}
|
||||
title={t('settings.voice.page.stt.modelDelete')}
|
||||
aria-label={t('settings.voice.page.stt.modelDelete')}
|
||||
>
|
||||
<Icon name="delete-bin" className="h-4 w-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : model.downloading ? (
|
||||
<span className="flex items-center gap-1.5">
|
||||
<Icon name="loader-4" className="h-3.5 w-3.5 animate-spin text-muted-foreground" />
|
||||
<span className="typography-ui-compact tabular-nums text-muted-foreground">
|
||||
{typeof model.downloadProgress === 'number' ? `${model.downloadProgress}%` : ''}
|
||||
</span>
|
||||
</span>
|
||||
) : (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
className="h-6 w-6 p-0"
|
||||
disabled={requestingId !== null}
|
||||
onClick={() => { void request(model.id, 'POST'); }}
|
||||
title={t('settings.voice.page.stt.modelDownload')}
|
||||
aria-label={t('settings.voice.page.stt.modelDownload')}
|
||||
>
|
||||
<Icon name="download" className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
{model.downloadError ? (
|
||||
<span className="typography-meta text-[var(--status-error)]">{model.downloadError}</span>
|
||||
) : null}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -424,6 +481,12 @@ export const VoiceSettings: React.FC = () => {
|
||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||
const setSayVoice = useConfigStore((state) => state.setSayVoice);
|
||||
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
|
||||
const localTtsModelId = useConfigStore((state) => state.localTtsModelId);
|
||||
const setLocalTtsModelId = useConfigStore((state) => state.setLocalTtsModelId);
|
||||
const localTtsModels = useLocalTtsModels();
|
||||
const localTtsVoiceOptions = useMemo(() => buildLocalTtsVoiceOptions(localTtsModels.models), [localTtsModels.models]);
|
||||
const ttsFollowTextLanguage = useConfigStore((state) => state.ttsFollowTextLanguage);
|
||||
const setTtsFollowTextLanguage = useConfigStore((state) => state.setTtsFollowTextLanguage);
|
||||
const setLocalTtsVoiceId = useConfigStore((state) => state.setLocalTtsVoiceId);
|
||||
const { speak: speakLocalTts, stop: stopLocalTts, isPlaying: isLocalTtsPlaying, error: localTtsError } = useLocalTTS();
|
||||
|
||||
@@ -432,13 +495,14 @@ export const VoiceSettings: React.FC = () => {
|
||||
stopLocalTts();
|
||||
return;
|
||||
}
|
||||
const voiceLabel = KOKORO_VOICE_OPTIONS.find((v) => v.id === localTtsVoiceId)?.label
|
||||
const voiceLabel = localTtsVoiceOptions.find((v) => v.modelId === localTtsModelId && v.speakerId === localTtsVoiceId)?.label
|
||||
?? String(localTtsVoiceId);
|
||||
void speakLocalTts(t('settings.voice.page.preview.voiceLine', { voiceName: voiceLabel }), {
|
||||
model: localTtsModelId,
|
||||
speakerId: localTtsVoiceId,
|
||||
speed: useConfigStore.getState().speechRate,
|
||||
});
|
||||
}, [isLocalTtsPlaying, localTtsVoiceId, speakLocalTts, stopLocalTts, t]);
|
||||
}, [isLocalTtsPlaying, localTtsModelId, localTtsVoiceId, localTtsVoiceOptions, speakLocalTts, stopLocalTts, t]);
|
||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||
const setBrowserVoice = useConfigStore((state) => state.setBrowserVoice);
|
||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||
@@ -959,24 +1023,39 @@ export const VoiceSettings: React.FC = () => {
|
||||
)}
|
||||
|
||||
{/* Local (Kokoro) TTS model status */}
|
||||
{voiceProvider === 'local' && <LocalTtsModelStatus />}
|
||||
{voiceProvider === 'local' && <LocalTtsModelStatus {...localTtsModels} />}
|
||||
|
||||
{(voiceProvider === 'local' || voiceProvider === 'say') && (
|
||||
<SettingsCheckboxRow
|
||||
checked={ttsFollowTextLanguage}
|
||||
onChange={setTtsFollowTextLanguage}
|
||||
label={t('settings.voice.page.field.followTextLanguage')}
|
||||
ariaLabel={t('settings.voice.page.field.followTextLanguageAria')}
|
||||
info={t('settings.voice.page.field.followTextLanguageInfo')}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Voice Selection */}
|
||||
<SettingsFieldRow label={t('settings.voice.page.field.voice')}>
|
||||
{voiceProvider === 'local' && (
|
||||
<>
|
||||
<Select
|
||||
value={String(localTtsVoiceId)}
|
||||
onValueChange={(value) => setLocalTtsVoiceId(Number.parseInt(value, 10) || 0)}
|
||||
value={localTtsVoiceKey(localTtsModelId, localTtsVoiceId)}
|
||||
onValueChange={(value) => {
|
||||
const option = localTtsVoiceOptions.find((v) => localTtsVoiceKey(v.modelId, v.speakerId) === value);
|
||||
if (!option) return;
|
||||
setLocalTtsModelId(option.modelId);
|
||||
setLocalTtsVoiceId(option.speakerId);
|
||||
}}
|
||||
>
|
||||
<SelectTrigger size={SETTINGS_SELECT_SIZE} className={SETTINGS_SELECT_ROW_TRIGGER_CLASS}>
|
||||
<SelectValue placeholder={t('settings.voice.page.field.selectVoicePlaceholder')}>
|
||||
{(value) => KOKORO_VOICE_OPTIONS.find((v) => String(v.id) === value)?.label ?? value}
|
||||
{(value) => localTtsVoiceOptions.find((v) => localTtsVoiceKey(v.modelId, v.speakerId) === value)?.label ?? value}
|
||||
</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{KOKORO_VOICE_OPTIONS.map((v) => (
|
||||
<SelectItem key={v.id} value={String(v.id)}>{v.label}</SelectItem>
|
||||
{localTtsVoiceOptions.map((v) => (
|
||||
<SelectItem key={localTtsVoiceKey(v.modelId, v.speakerId)} value={localTtsVoiceKey(v.modelId, v.speakerId)}>{v.label}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
Reference in New Issue
Block a user