Files
openchamber/packages/ui/src/hooks/useSayTTS.ts
T
Bohdan Triapitsyn 391f938334 feat(voice): match local and macOS voices to the language of the text
Text-to-speech picked one voice regardless of what language a reply was in.
A dependency-free language detector (script, marker letters, function words)
now decides the language of the whole message once; with the new
"Match the voice to the language of the text" setting the local provider
switches to a catalog model for that language (Kokoro zh/en and Piper models
for 12 languages, downloaded on first use like the existing model) and macOS
say switches to an installed voice whose locale matches. The local voice
picker lists voices of every installed model, and the settings show which
language models are on disk.

The Ukrainian Piper medium build is a character-level model that sherpa-onnx
turns into noise, so the espeak-based Lada build is used instead.

Claude-Session: https://claude.ai/code/session_017TK5JAYDfT3Fotc23UEg98
2026-08-30 02:24:22 +03:00

313 lines
8.8 KiB
TypeScript

/**
* useSayTTS Hook
*
* React hook for macOS 'say' command text-to-speech playback.
* Uses the native macOS speech synthesis via server API.
* Uses Web Audio API for playback (better iOS Safari support).
*
* @example
* ```typescript
* const { speak, isPlaying, stop, isAvailable } = useSayTTS();
*
* // Speak text
* await speak('Hello, this is a test');
*
* // Stop playback
* stop();
* ```
*/
import { useCallback, useEffect, useRef, useState } from 'react';
import { runtimeFetch } from '@/lib/runtime-fetch';
interface SayTTSStatusCache {
available: boolean;
voices: Array<{ name: string; locale: string }>;
checkedAt: number;
}
interface UseSayTTSOptions {
enabled?: boolean;
}
const SAY_TTS_STATUS_TTL_MS = 30000;
let sayTTSStatusCache: SayTTSStatusCache | null = null;
let sayTTSStatusRequest: Promise<SayTTSStatusCache> | null = null;
async function getSayTTSStatus(): Promise<SayTTSStatusCache> {
const now = Date.now();
if (sayTTSStatusCache && now - sayTTSStatusCache.checkedAt < SAY_TTS_STATUS_TTL_MS) {
return sayTTSStatusCache;
}
if (sayTTSStatusRequest) {
return sayTTSStatusRequest;
}
sayTTSStatusRequest = (async () => {
try {
const response = await runtimeFetch('/api/tts/say/status');
if (!response.ok) {
const unavailableStatus: SayTTSStatusCache = {
available: false,
voices: [],
checkedAt: Date.now(),
};
sayTTSStatusCache = unavailableStatus;
return unavailableStatus;
}
const data = await response.json();
const nextStatus: SayTTSStatusCache = {
available: Boolean(data.available),
voices: Array.isArray(data.voices) ? data.voices : [],
checkedAt: Date.now(),
};
sayTTSStatusCache = nextStatus;
return nextStatus;
} catch {
const unavailableStatus: SayTTSStatusCache = {
available: false,
voices: [],
checkedAt: Date.now(),
};
sayTTSStatusCache = unavailableStatus;
return unavailableStatus;
} finally {
sayTTSStatusRequest = null;
}
})();
return sayTTSStatusRequest;
}
export interface UseSayTTSReturn {
/** Whether TTS is currently playing */
isPlaying: boolean;
/** Whether the macOS say command is available */
isAvailable: boolean;
/** Available voices */
voices: Array<{ name: string; locale: string }>;
/** Current error if any */
error: string | null;
/** Speak the given text */
speak: (text: string, options?: SpeakOptions) => Promise<void>;
/** Stop current playback */
stop: () => void;
/** Check if service is available */
checkAvailability: () => Promise<boolean>;
/** Unlock audio for mobile Safari - call this on user gesture */
unlockAudio: () => Promise<void>;
}
interface SpeakOptions {
/** Voice to use (defaults to Samantha) */
voice?: string;
/** Speech rate in words per minute (defaults to 200) */
rate?: number;
/** `'auto'`: the server switches to a voice that speaks the text's language. */
language?: 'auto';
/** Callback when playback starts */
onStart?: () => void;
/** Callback when playback ends */
onEnd?: () => void;
/** Callback on error */
onError?: (error: string) => void;
}
// Shared AudioContext for Web Audio API playback (better iOS support)
let sharedAudioContext: AudioContext | null = null;
function getAudioContext(): AudioContext {
if (!sharedAudioContext) {
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
}
return sharedAudioContext;
}
export function useSayTTS(options: UseSayTTSOptions = {}): UseSayTTSReturn {
const enabled = options.enabled ?? true;
const [isPlaying, setIsPlaying] = useState(false);
const [isAvailable, setIsAvailable] = useState(false);
const [voices, setVoices] = useState<Array<{ name: string; locale: string }>>([]);
const [error, setError] = useState<string | null>(null);
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
const abortControllerRef = useRef<AbortController | null>(null);
// Unlock audio for mobile Safari - must be called within user gesture
const unlockAudio = useCallback(async (): Promise<void> => {
try {
// Get or create AudioContext
const ctx = getAudioContext();
// Resume if suspended (required for iOS Safari)
if (ctx.state === 'suspended') {
await ctx.resume();
console.log('[useSayTTS] AudioContext resumed');
}
// Play a tiny silent buffer to fully unlock
const buffer = ctx.createBuffer(1, 1, 22050);
const source = ctx.createBufferSource();
source.buffer = buffer;
source.connect(ctx.destination);
source.start(0);
console.log('[useSayTTS] Audio unlocked for mobile playback');
} catch (err) {
console.error('[useSayTTS] Failed to unlock audio:', err);
}
}, []);
// Check if macOS say is available
const checkAvailability = useCallback(async (): Promise<boolean> => {
if (!enabled) {
setIsAvailable(false);
setVoices([]);
return false;
}
try {
const status = await getSayTTSStatus();
setIsAvailable(status.available);
setVoices(status.voices);
return status.available;
} catch {
setIsAvailable(false);
setVoices([]);
return false;
}
}, [enabled]);
// Check availability on mount
useEffect(() => {
void checkAvailability();
}, [checkAvailability]);
// Stop current playback
const stop = useCallback(() => {
if (audioSourceRef.current) {
try {
audioSourceRef.current.stop();
} catch {
// Already stopped
}
audioSourceRef.current = null;
}
if (abortControllerRef.current) {
abortControllerRef.current.abort();
abortControllerRef.current = null;
}
setIsPlaying(false);
}, []);
// Speak text using macOS say
const speak = useCallback(async (text: string, options?: SpeakOptions): Promise<void> => {
// Stop any existing playback
stop();
if (!text.trim()) {
setError('No text to speak');
options?.onError?.('No text to speak');
return;
}
setError(null);
try {
// Create abort controller for this request
abortControllerRef.current = new AbortController();
// Fetch audio from server
const response = await runtimeFetch('/api/tts/say/speak', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
text: text.trim(),
voice: options?.voice || 'Samantha',
rate: options?.rate || 200,
language: options?.language,
}),
signal: abortControllerRef.current.signal,
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
throw new Error(errorData.error || `HTTP ${response.status}`);
}
// Get audio data from response
const audioBlob = await response.blob();
const arrayBuffer = await audioBlob.arrayBuffer();
console.log('[useSayTTS] Received audio:', audioBlob.size, 'bytes');
// Use Web Audio API for playback (same as useServerTTS)
const ctx = getAudioContext();
// Resume context if suspended
if (ctx.state === 'suspended') {
await ctx.resume();
console.log('[useSayTTS] AudioContext resumed before playback');
}
// Decode audio data
console.log('[useSayTTS] Decoding audio data...');
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
// Create source node
const source = ctx.createBufferSource();
source.buffer = audioBuffer;
source.connect(ctx.destination);
audioSourceRef.current = source;
// Set up event handlers
source.onended = () => {
console.log('[useSayTTS] Audio playback ended');
setIsPlaying(false);
audioSourceRef.current = null;
options?.onEnd?.();
};
// Start playback
console.log('[useSayTTS] Starting audio playback via Web Audio API...');
setIsPlaying(true);
options?.onStart?.();
source.start(0);
} catch (err) {
if ((err as Error).name === 'AbortError') {
return;
}
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
console.error('[useSayTTS] Error:', errorMsg);
setError(errorMsg);
options?.onError?.(errorMsg);
setIsPlaying(false);
}
}, [stop]);
// Cleanup on unmount
useEffect(() => {
return () => {
stop();
};
}, [stop]);
return {
isPlaying,
isAvailable,
voices,
error,
speak,
stop,
checkAvailability,
unlockAudio,
};
}