feat(voice): add voice input/output support with multiple providers (#281)
* feat(voice): add voice input/output support with multiple providers - Add BrowserVoiceButton component for Web Speech API voice input - Add VoiceProvider context for managing voice state across the app - Add TTS (Text-to-Speech) support with browser, macOS Say, and OpenAI providers - Add message TTS buttons to read assistant messages aloud - Add VoiceSettings page in OpenChamber settings - Add server endpoints for TTS and summarization services - Include slider component for voice rate/pitch/volume controls - Add hidden session support for background voice operations - Add Caddyfile for HTTPS support (required for microphone access) * fix: Build errors fixed and removed outdated ElevenLabs test code. * refactor(voice): use zen API with gpt-5-nano for TTS summarization Replace the hidden session + OpenCode SDK approach with direct calls to the opencode.ai zen API (same pattern used for commit message and PR description generation). - Rewrite summarization-service.js to call zen/v1/responses with gpt-5-nano - Remove hidden session logic (hiddenSession.ts, sessionStore filtering) - Remove summarizeModel setting and model selector from VoiceSettings - Simplify client-side summarize.ts to no longer pass model params - Clean up callers in useMessageTTS and useBrowserVoice * fix(voice): remove false 'voice not supported' warning in settings Mobile Safari does support voice but the isSupported check was incorrectly flagging it. Remove the warning banner entirely. * feat(voice): add configurable summary length limit for TTS output Add a slider (50-2000 chars) in voice settings to control max summary length. The limit is passed through the summarize endpoint and speak endpoint to the zen API prompt, with token budget scaled accordingly. * fix(voice): add diagnostic logging and sanitize TTS fallback Add console logging throughout the summarization flow (client + server) to trace why text may not be summarized. Fix silent error swallowing in /api/tts/speak. Always apply sanitizeForTTS even when summarization is disabled so raw markdown/code is never spoken verbatim. * fix(voice): fix token budget starving model of output tokens max_output_tokens includes both reasoning and output tokens. With effort:'low', reasoning alone consumes ~128 tokens, so a budget of 100 left zero tokens for the actual summary text. Use a fixed 1000 token budget (matching commit message generation) and control output length via the prompt's character limit instruction instead. * chore(voice): remove diagnostic logging from summarization flow * fix(voice): don't request mic permission on mobile page load Remove the useEffect that pre-requested microphone permission when the BrowserVoiceButton component mounted on mobile. This caused an unwanted permission prompt immediately on page load before the user tapped the mic icon. Permission is now only requested on explicit user interaction. * fix(voice): remove unused BrowserVoiceButton binding * fix(voice): desktop mic flow + non-continuous draft mode * fix(voice): stabilize continuous loop and polish controls * feat(settings): mark voice section experimental --------- Co-authored-by: Bohdan Triapitsyn <artmore@protonmail.com>
This commit is contained in:
gsxdsm
committed by
GitHub
GitHub
co-authored by
Bohdan Triapitsyn
Bohdan Triapitsyn
parent
6776ac31c2
commit
1ed5316ac7
@@ -0,0 +1,792 @@
|
||||
/**
|
||||
* useBrowserVoice Hook
|
||||
*
|
||||
* React hook for browser-based voice chat integration.
|
||||
* Manages speech recognition, AI message sending, and speech synthesis.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const {
|
||||
* status,
|
||||
* isSupported,
|
||||
* language,
|
||||
* setLanguage,
|
||||
* startVoice,
|
||||
* stopVoice,
|
||||
* prepareVoice,
|
||||
* isMobile,
|
||||
* } = useBrowserVoice();
|
||||
*
|
||||
* // Start voice mode
|
||||
* startVoice();
|
||||
*
|
||||
* // Change language
|
||||
* setLanguage('es-ES');
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { useSessionStore } from '@/stores/useSessionStore';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useServerTTS } from './useServerTTS';
|
||||
import { useSayTTS } from './useSayTTS';
|
||||
import { summarizeText, shouldSummarize, sanitizeForTTS } from '@/lib/voice/summarize';
|
||||
|
||||
export type BrowserVoiceStatus = 'idle' | 'listening' | 'processing' | 'speaking' | 'error';
|
||||
|
||||
export interface UseBrowserVoiceReturn {
|
||||
/** Current voice status */
|
||||
status: BrowserVoiceStatus;
|
||||
/** Whether browser voice is supported */
|
||||
isSupported: boolean;
|
||||
/** Error message if any */
|
||||
error: string | null;
|
||||
/** Current language for recognition/synthesis */
|
||||
language: string;
|
||||
/** Set language for voice operations */
|
||||
setLanguage: (lang: string) => void;
|
||||
/** Start voice mode (listening) */
|
||||
startVoice: () => void;
|
||||
/** Stop voice mode */
|
||||
stopVoice: () => void;
|
||||
/** Whether conversation mode is active */
|
||||
conversationMode: boolean;
|
||||
/** Toggle conversation mode */
|
||||
toggleConversationMode: () => void;
|
||||
/** Prepare voice for mobile (request permission) */
|
||||
prepareVoice: () => Promise<boolean>;
|
||||
/** Whether the device is mobile */
|
||||
isMobile: boolean;
|
||||
/** Current voice provider */
|
||||
voiceProvider: 'browser' | 'openai' | 'say';
|
||||
}
|
||||
|
||||
// Storage key for persisting language preference
|
||||
const LANGUAGE_STORAGE_KEY = 'browserVoiceLanguage';
|
||||
// Storage key for persisting conversation mode preference
|
||||
const CONVERSATION_MODE_STORAGE_KEY = 'browserVoiceConversationMode';
|
||||
const LANGUAGE_CHANGE_EVENT = 'openchamber:voice-language-changed';
|
||||
const CONVERSATION_MODE_CHANGE_EVENT = 'openchamber:voice-conversation-mode-changed';
|
||||
const FINAL_TRANSCRIPT_SETTLE_MS = 1200;
|
||||
const DEVICE_CHANGE_RESTART_DELAY_MS = 700;
|
||||
const BLOCKED_SPEECH_LANGUAGES = new Set(['ru', 'ru-RU']);
|
||||
|
||||
const sanitizeSpeechLanguage = (lang: string): string => {
|
||||
const normalized = (lang || '').trim();
|
||||
if (!normalized) {
|
||||
return 'en-US';
|
||||
}
|
||||
const base = normalized.split('-')[0].toLowerCase();
|
||||
if (BLOCKED_SPEECH_LANGUAGES.has(normalized) || BLOCKED_SPEECH_LANGUAGES.has(base)) {
|
||||
return 'en-US';
|
||||
}
|
||||
return normalized;
|
||||
};
|
||||
|
||||
/**
|
||||
* Hook for managing browser-based voice conversations
|
||||
*/
|
||||
export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
const [status, setStatus] = useState<BrowserVoiceStatus>('idle');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [language, setLanguageState] = useState<string>(() => {
|
||||
// Try to load from localStorage, fallback to navigator.language
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem(LANGUAGE_STORAGE_KEY);
|
||||
if (saved) return sanitizeSpeechLanguage(saved);
|
||||
}
|
||||
return sanitizeSpeechLanguage(navigator.language || 'en-US');
|
||||
});
|
||||
const [conversationMode, setConversationModeState] = useState<boolean>(() => {
|
||||
// Try to load from localStorage, default to false
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem(CONVERSATION_MODE_STORAGE_KEY);
|
||||
return saved === 'true';
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
const isSupported = browserVoiceService.isSupported();
|
||||
|
||||
// Mobile detection
|
||||
const isMobile = useMemo(() => {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
return /iphone|ipad|ipod|android|mobile|webos|blackberry|iemobile|opera mini/i.test(userAgent);
|
||||
}, []);
|
||||
|
||||
// Refs for managing async operations
|
||||
const isActiveRef = useRef(false);
|
||||
const processingMessageRef = useRef(false);
|
||||
const lastTranscriptRef = useRef('');
|
||||
const messagesRef = useRef<Map<string, { info: { role: string }; parts: Array<{ type: string; text?: string }> }>>(new Map());
|
||||
const pendingResumeOnVisibleRef = useRef(false);
|
||||
const pendingFinalTranscriptRef = useRef('');
|
||||
const finalTranscriptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const deviceChangeRestartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Store access
|
||||
const currentSessionId = useSessionStore((s) => s.currentSessionId);
|
||||
const sendMessage = useSessionStore((s) => s.sendMessage);
|
||||
const setPendingInputText = useSessionStore((s) => s.setPendingInputText);
|
||||
const messages = useSessionStore((s) => s.messages);
|
||||
const createSession = useSessionStore((s) => s.createSession);
|
||||
const { currentProviderId, currentModelId, currentAgentName, voiceProvider, speechRate, speechPitch, speechVolume, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold } = useConfigStore();
|
||||
|
||||
// Server TTS for mobile (bypasses Safari audio restrictions)
|
||||
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS();
|
||||
|
||||
// macOS Say TTS
|
||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable, unlockAudio: unlockSayTTSAudio } = useSayTTS();
|
||||
|
||||
// Update messages ref when messages change
|
||||
useEffect(() => {
|
||||
if (currentSessionId) {
|
||||
const sessionMessages = messages.get(currentSessionId);
|
||||
if (sessionMessages) {
|
||||
messagesRef.current = new Map(sessionMessages.map(m => [m.info.id, m]));
|
||||
}
|
||||
}
|
||||
}, [messages, currentSessionId]);
|
||||
|
||||
// Stop voice when session changes to prevent microphone from staying active
|
||||
// This ensures voice mode doesn't carry over between sessions
|
||||
const prevSessionIdRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (prevSessionIdRef.current !== null && prevSessionIdRef.current !== currentSessionId) {
|
||||
// Session changed - stop any active voice session
|
||||
if (isActiveRef.current) {
|
||||
console.log('[useBrowserVoice] Session changed, stopping voice');
|
||||
isActiveRef.current = false;
|
||||
processingMessageRef.current = false;
|
||||
browserVoiceService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
setStatus('idle');
|
||||
setError(null);
|
||||
}
|
||||
}
|
||||
prevSessionIdRef.current = currentSessionId;
|
||||
}, [currentSessionId]);
|
||||
|
||||
// Persist language preference
|
||||
const setLanguage = useCallback((lang: string) => {
|
||||
const nextLang = sanitizeSpeechLanguage(lang);
|
||||
setLanguageState(nextLang);
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem(LANGUAGE_STORAGE_KEY, nextLang);
|
||||
window.dispatchEvent(new CustomEvent<string>(LANGUAGE_CHANGE_EVENT, { detail: nextLang }));
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof window === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleLanguageEvent = (event: Event) => {
|
||||
const customEvent = event as CustomEvent<string>;
|
||||
const nextLang = sanitizeSpeechLanguage(customEvent.detail || localStorage.getItem(LANGUAGE_STORAGE_KEY) || 'en-US');
|
||||
setLanguageState((prev) => (prev === nextLang ? prev : nextLang));
|
||||
};
|
||||
|
||||
const handleStorage = (event: StorageEvent) => {
|
||||
if (event.key !== LANGUAGE_STORAGE_KEY || !event.newValue) {
|
||||
return;
|
||||
}
|
||||
const nextLang = sanitizeSpeechLanguage(event.newValue);
|
||||
setLanguageState((prev) => (prev === nextLang ? prev : nextLang));
|
||||
};
|
||||
|
||||
window.addEventListener(LANGUAGE_CHANGE_EVENT, handleLanguageEvent as EventListener);
|
||||
window.addEventListener('storage', handleStorage);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener(LANGUAGE_CHANGE_EVENT, handleLanguageEvent as EventListener);
|
||||
window.removeEventListener('storage', handleStorage);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Toggle conversation mode
|
||||
const toggleConversationMode = useCallback(() => {
|
||||
setConversationModeState((prev) => {
|
||||
const next = !prev;
|
||||
browserVoiceService.setConversationMode(next);
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem(CONVERSATION_MODE_STORAGE_KEY, String(next));
|
||||
window.dispatchEvent(new CustomEvent<boolean>(CONVERSATION_MODE_CHANGE_EVENT, { detail: next }));
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof window === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleConversationModeEvent = (event: Event) => {
|
||||
const customEvent = event as CustomEvent<boolean>;
|
||||
const detail = customEvent.detail;
|
||||
if (typeof detail !== 'boolean') {
|
||||
return;
|
||||
}
|
||||
setConversationModeState((prev) => (prev === detail ? prev : detail));
|
||||
};
|
||||
|
||||
const handleStorage = (event: StorageEvent) => {
|
||||
if (event.key !== CONVERSATION_MODE_STORAGE_KEY || event.newValue == null) {
|
||||
return;
|
||||
}
|
||||
const next = event.newValue === 'true';
|
||||
setConversationModeState((prev) => (prev === next ? prev : next));
|
||||
};
|
||||
|
||||
window.addEventListener(CONVERSATION_MODE_CHANGE_EVENT, handleConversationModeEvent as EventListener);
|
||||
window.addEventListener('storage', handleStorage);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener(CONVERSATION_MODE_CHANGE_EVENT, handleConversationModeEvent as EventListener);
|
||||
window.removeEventListener('storage', handleStorage);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Initialize conversation mode in service on mount
|
||||
useEffect(() => {
|
||||
browserVoiceService.setConversationMode(conversationMode);
|
||||
}, [conversationMode]);
|
||||
|
||||
// Refs for callbacks to avoid circular dependencies
|
||||
const handleSpeechErrorRef = useRef<((errorMsg: string) => void) | null>(null);
|
||||
const handleSpeechResultRef = useRef<((text: string, isFinal: boolean) => Promise<void>) | null>(null);
|
||||
|
||||
// Handle speech recognition error
|
||||
const handleSpeechError = useCallback((errorMsg: string) => {
|
||||
// Ignore errors if we've already stopped voice mode
|
||||
if (!isActiveRef.current) {
|
||||
console.log('[useBrowserVoice] Ignoring error after voice stopped:', errorMsg);
|
||||
return;
|
||||
}
|
||||
|
||||
const normalizedError = errorMsg.toLowerCase();
|
||||
if (normalizedError.includes('aborted')) {
|
||||
console.log('[useBrowserVoice] Ignoring non-fatal aborted error');
|
||||
setError(null);
|
||||
setStatus('listening');
|
||||
return;
|
||||
}
|
||||
|
||||
const isHidden = typeof document !== 'undefined' && document.visibilityState !== 'visible';
|
||||
const isPermissionStyleError =
|
||||
normalizedError.includes('permission') ||
|
||||
normalizedError.includes('not allowed') ||
|
||||
normalizedError.includes('service not allowed');
|
||||
|
||||
if (isHidden && isPermissionStyleError && conversationMode) {
|
||||
console.log('[useBrowserVoice] Suppressing permission error while app hidden; will resume on visibility');
|
||||
pendingResumeOnVisibleRef.current = true;
|
||||
setError(null);
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
console.error('[useBrowserVoice] Recognition error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
|
||||
// Auto-recover from certain errors
|
||||
if (!errorMsg.includes('permission') && !errorMsg.includes('not allowed')) {
|
||||
setTimeout(() => {
|
||||
if (isActiveRef.current) {
|
||||
setStatus('listening');
|
||||
setError(null);
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
}, [language, conversationMode]);
|
||||
|
||||
// Update the ref when handleSpeechError changes
|
||||
useEffect(() => {
|
||||
handleSpeechErrorRef.current = handleSpeechError;
|
||||
}, [handleSpeechError]);
|
||||
|
||||
const processFinalTranscript = useCallback(async (finalText: string) => {
|
||||
if (!finalText.trim() || !isActiveRef.current) return;
|
||||
|
||||
// Prevent duplicate processing of same transcript
|
||||
if (finalText.trim() === lastTranscriptRef.current) return;
|
||||
lastTranscriptRef.current = finalText.trim();
|
||||
|
||||
// Check if provider and model are configured
|
||||
if (!currentProviderId || !currentModelId) {
|
||||
setError('No provider or model configured. Please configure them in settings.');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop listening while processing
|
||||
browserVoiceService.stopListening();
|
||||
|
||||
// Non-continuous mode: fill chat input only, do not auto-send.
|
||||
if (!conversationMode) {
|
||||
setPendingInputText(finalText.trim(), 'replace');
|
||||
processingMessageRef.current = false;
|
||||
isActiveRef.current = false;
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus('processing');
|
||||
processingMessageRef.current = true;
|
||||
|
||||
try {
|
||||
// Create session if none exists
|
||||
let sessionId = currentSessionId;
|
||||
if (!sessionId) {
|
||||
console.log('[useBrowserVoice] No active session, creating new session...');
|
||||
const newSession = await createSession();
|
||||
if (!newSession) {
|
||||
setError('Failed to create session');
|
||||
setStatus('error');
|
||||
processingMessageRef.current = false;
|
||||
return;
|
||||
}
|
||||
sessionId = newSession.id;
|
||||
console.log('[useBrowserVoice] Created new session:', sessionId);
|
||||
}
|
||||
|
||||
// Send message to AI
|
||||
await sendMessage(
|
||||
finalText.trim(),
|
||||
currentProviderId,
|
||||
currentModelId,
|
||||
currentAgentName ?? undefined
|
||||
);
|
||||
|
||||
// Wait for AI response and speak it
|
||||
// We'll poll for new assistant messages
|
||||
const checkForResponse = async () => {
|
||||
if (!isActiveRef.current) return;
|
||||
|
||||
const sessionMessages = messagesRef.current;
|
||||
const assistantMessages = Array.from(sessionMessages.values())
|
||||
.filter(m => m.info.role === 'assistant')
|
||||
.sort((a, b) => {
|
||||
const aTime = (a.info as { time?: { created?: number } }).time?.created ?? 0;
|
||||
const bTime = (b.info as { time?: { created?: number } }).time?.created ?? 0;
|
||||
return bTime - aTime;
|
||||
});
|
||||
|
||||
if (assistantMessages.length > 0) {
|
||||
const latestMessage = assistantMessages[0];
|
||||
const textParts = latestMessage.parts
|
||||
.filter(p => p.type === 'text')
|
||||
.map(p => p.text)
|
||||
.join(' ');
|
||||
|
||||
if (textParts.trim()) {
|
||||
// Speak the response
|
||||
setStatus('speaking');
|
||||
try {
|
||||
// Summarize text if enabled and over threshold
|
||||
let textToSpeak = textParts;
|
||||
if (summarizeVoiceConversation && shouldSummarize(textParts, 'voice')) {
|
||||
console.log('[useBrowserVoice] Summarizing AI response before speaking...');
|
||||
textToSpeak = await summarizeText(textParts, {
|
||||
threshold: summarizeCharacterThreshold,
|
||||
});
|
||||
} else {
|
||||
// Still sanitize for TTS even when not summarizing
|
||||
textToSpeak = sanitizeForTTS(textParts);
|
||||
}
|
||||
|
||||
// Helper to restart listening after speech ends
|
||||
// Only auto-restart if conversation mode is enabled
|
||||
const restartListening = () => {
|
||||
if (isActiveRef.current && conversationMode) {
|
||||
const isHidden = typeof document !== 'undefined' && document.visibilityState !== 'visible';
|
||||
if (isHidden) {
|
||||
pendingResumeOnVisibleRef.current = true;
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus('listening');
|
||||
if (isMobile) {
|
||||
try {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} catch (err) {
|
||||
console.error('[useBrowserVoice] Failed to restart listening:', err);
|
||||
}
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} else {
|
||||
// In non-continuous mode, return to idle after AI responds
|
||||
isActiveRef.current = false;
|
||||
setStatus('idle');
|
||||
}
|
||||
};
|
||||
|
||||
// Use server TTS when OpenAI provider is selected and available
|
||||
if (voiceProvider === 'openai' && isServerTTSAvailable) {
|
||||
console.log('[useBrowserVoice] Using OpenAI server TTS with voice:', openaiVoice);
|
||||
await speakServerTTS(textToSpeak, {
|
||||
voice: openaiVoice,
|
||||
speed: speechRate,
|
||||
onStart: () => console.log('[useBrowserVoice] Server TTS started'),
|
||||
onEnd: () => {
|
||||
console.log('[useBrowserVoice] Server TTS ended');
|
||||
restartListening();
|
||||
},
|
||||
onError: (errorMsg) => {
|
||||
console.error('[useBrowserVoice] Server TTS error:', errorMsg);
|
||||
// Show error to user when OpenAI voice fails
|
||||
setError(`OpenAI voice failed: ${errorMsg}. Please check your OpenAI API key or switch to Browser voice.`);
|
||||
setStatus('error');
|
||||
restartListening();
|
||||
}
|
||||
});
|
||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||
// Use macOS 'say' command
|
||||
console.log('[useBrowserVoice] Using macOS Say TTS with voice:', sayVoice);
|
||||
// Convert speechRate (0.5-2.0) to words per minute (100-400)
|
||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||
await speakSayTTS(textToSpeak, {
|
||||
voice: sayVoice,
|
||||
rate: wordsPerMinute,
|
||||
onStart: () => console.log('[useBrowserVoice] Say TTS started'),
|
||||
onEnd: () => {
|
||||
console.log('[useBrowserVoice] Say TTS ended');
|
||||
restartListening();
|
||||
},
|
||||
onError: (errorMsg) => {
|
||||
console.error('[useBrowserVoice] Say TTS error:', errorMsg);
|
||||
restartListening();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Use browser TTS (desktop and mobile)
|
||||
// Pre-load voices and unlock audio context before speaking
|
||||
console.log('[useBrowserVoice] Using browser TTS');
|
||||
|
||||
// Warn user if they selected OpenAI but it's unavailable
|
||||
if (voiceProvider === 'openai' && !isServerTTSAvailable) {
|
||||
console.warn('[useBrowserVoice] OpenAI voice selected but unavailable, falling back to browser voice');
|
||||
setError('OpenAI voice unavailable (API key not configured). Using browser voice instead.');
|
||||
}
|
||||
|
||||
await browserVoiceService.waitForVoices();
|
||||
await browserVoiceService.resumeAudioContext();
|
||||
|
||||
await browserVoiceService.speakText(textToSpeak, language, () => {
|
||||
// When speech ends, go back to listening if still active
|
||||
restartListening();
|
||||
}, { rate: speechRate, pitch: speechPitch, volume: speechVolume, voiceName: browserVoice || undefined });
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Speech failed';
|
||||
|
||||
// Ignore errors if we've stopped voice (e.g., user cancelled during speech)
|
||||
if (!isActiveRef.current) {
|
||||
console.log('[useBrowserVoice] Ignoring speech error after voice stopped:', errorMsg);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error('[useBrowserVoice] Speech error:', errorMsg);
|
||||
|
||||
// Check for autoplay policy error
|
||||
if (errorMsg.includes('not-allowed') || errorMsg.includes('autoplay')) {
|
||||
setError('Audio blocked by browser. Please click the voice button again to enable audio.');
|
||||
}
|
||||
|
||||
// Only restart listening if conversation mode is enabled
|
||||
if (conversationMode) {
|
||||
setStatus('listening');
|
||||
if (isMobile) {
|
||||
try {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} catch (restartErr) {
|
||||
console.error('[useBrowserVoice] Failed to restart listening after speech error:', restartErr);
|
||||
}
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} else {
|
||||
// In non-continuous mode, return to idle after error
|
||||
isActiveRef.current = false;
|
||||
setStatus('idle');
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Check again in 500ms
|
||||
setTimeout(checkForResponse, 500);
|
||||
};
|
||||
|
||||
// Start checking for response after a short delay
|
||||
setTimeout(checkForResponse, 1000);
|
||||
|
||||
} catch (err) {
|
||||
console.error('[useBrowserVoice] Send message error:', err);
|
||||
setError(err instanceof Error ? err.message : 'Failed to send message');
|
||||
setStatus('error');
|
||||
processingMessageRef.current = false;
|
||||
}
|
||||
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode]);
|
||||
|
||||
// Handle speech recognition result
|
||||
const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
|
||||
if (!isActiveRef.current) return;
|
||||
const normalized = text.trim();
|
||||
if (!isFinal || !normalized) return;
|
||||
|
||||
pendingFinalTranscriptRef.current = normalized;
|
||||
|
||||
if (finalTranscriptTimerRef.current) {
|
||||
clearTimeout(finalTranscriptTimerRef.current);
|
||||
}
|
||||
|
||||
finalTranscriptTimerRef.current = setTimeout(() => {
|
||||
finalTranscriptTimerRef.current = null;
|
||||
const transcript = pendingFinalTranscriptRef.current.trim();
|
||||
pendingFinalTranscriptRef.current = '';
|
||||
if (!transcript) return;
|
||||
void processFinalTranscript(transcript);
|
||||
}, FINAL_TRANSCRIPT_SETTLE_MS);
|
||||
}, [processFinalTranscript]);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof document === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleVisibilityChange = () => {
|
||||
if (document.visibilityState !== 'visible') {
|
||||
return;
|
||||
}
|
||||
if (!pendingResumeOnVisibleRef.current) {
|
||||
return;
|
||||
}
|
||||
if (!isActiveRef.current || !conversationMode) {
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
setStatus('listening');
|
||||
try {
|
||||
if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to resume voice';
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener('visibilitychange', handleVisibilityChange);
|
||||
return () => {
|
||||
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
||||
};
|
||||
}, [conversationMode, isMobile, language]);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof navigator === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const mediaDevices = navigator.mediaDevices;
|
||||
if (!mediaDevices || typeof mediaDevices.addEventListener !== 'function') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleDeviceChange = () => {
|
||||
if (!isActiveRef.current || status !== 'listening') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
}
|
||||
|
||||
deviceChangeRestartTimerRef.current = setTimeout(() => {
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
if (!isActiveRef.current || status !== 'listening') {
|
||||
return;
|
||||
}
|
||||
|
||||
const isHidden = typeof document !== 'undefined' && document.visibilityState !== 'visible';
|
||||
if (isHidden) {
|
||||
pendingResumeOnVisibleRef.current = true;
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
browserVoiceService.stopListening();
|
||||
if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
}, DEVICE_CHANGE_RESTART_DELAY_MS);
|
||||
};
|
||||
|
||||
mediaDevices.addEventListener('devicechange', handleDeviceChange);
|
||||
|
||||
return () => {
|
||||
mediaDevices.removeEventListener('devicechange', handleDeviceChange);
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [isMobile, language, status]);
|
||||
|
||||
// Update the ref when handleSpeechResult changes
|
||||
useEffect(() => {
|
||||
handleSpeechResultRef.current = handleSpeechResult;
|
||||
}, [handleSpeechResult]);
|
||||
|
||||
// Prepare voice for mobile (request permission)
|
||||
const prepareVoice = useCallback(async (): Promise<boolean> => {
|
||||
if (!isSupported) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await browserVoiceService.prepareListening();
|
||||
return true;
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Microphone permission denied';
|
||||
setError(errorMsg);
|
||||
return false;
|
||||
}
|
||||
}, [isSupported]);
|
||||
|
||||
// Start voice mode
|
||||
const startVoice = useCallback(async () => {
|
||||
if (!isSupported) {
|
||||
setError('Browser voice not supported');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!currentSessionId) {
|
||||
setError('No active session');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
|
||||
isActiveRef.current = true;
|
||||
lastTranscriptRef.current = '';
|
||||
setError(null);
|
||||
setStatus('listening');
|
||||
|
||||
// On mobile, use sync path to ensure SpeechRecognition.start() is called
|
||||
// within the same user gesture context (required by iOS Safari)
|
||||
// Also unlock audio immediately for TTS playback later
|
||||
if (isMobile) {
|
||||
try {
|
||||
// Unlock audio context synchronously within user gesture
|
||||
browserVoiceService.unlockAudio().catch(() => {
|
||||
// Audio unlock failed, but continue anyway
|
||||
});
|
||||
// Also unlock server TTS audio for mobile Safari (OpenAI)
|
||||
unlockServerTTSAudio().catch(() => {
|
||||
// Server TTS unlock failed, but continue anyway
|
||||
});
|
||||
// Also unlock Say TTS audio for mobile Safari (macOS Say)
|
||||
unlockSayTTSAudio().catch(() => {
|
||||
// Say TTS unlock failed, but continue anyway
|
||||
});
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResult, handleSpeechError);
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
|
||||
console.error('[useBrowserVoice] Mobile voice start error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
} else {
|
||||
// Desktop can use async path with permission check
|
||||
try {
|
||||
await browserVoiceService.startListening(language, handleSpeechResult, handleSpeechError);
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
|
||||
console.error('[useBrowserVoice] Desktop voice start error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
}
|
||||
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio]);
|
||||
|
||||
// Stop voice mode
|
||||
const stopVoice = useCallback(() => {
|
||||
isActiveRef.current = false;
|
||||
processingMessageRef.current = false;
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
pendingFinalTranscriptRef.current = '';
|
||||
if (finalTranscriptTimerRef.current) {
|
||||
clearTimeout(finalTranscriptTimerRef.current);
|
||||
finalTranscriptTimerRef.current = null;
|
||||
}
|
||||
browserVoiceService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
stopServerTTS(); // Also stop server TTS if playing
|
||||
stopSayTTS(); // Also stop Say TTS if playing
|
||||
setStatus('idle');
|
||||
setError(null);
|
||||
}, [stopServerTTS, stopSayTTS]);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
isActiveRef.current = false;
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
pendingFinalTranscriptRef.current = '';
|
||||
if (finalTranscriptTimerRef.current) {
|
||||
clearTimeout(finalTranscriptTimerRef.current);
|
||||
finalTranscriptTimerRef.current = null;
|
||||
}
|
||||
browserVoiceService.setConversationMode(false);
|
||||
browserVoiceService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
status,
|
||||
isSupported,
|
||||
error,
|
||||
language,
|
||||
setLanguage,
|
||||
startVoice,
|
||||
stopVoice,
|
||||
conversationMode,
|
||||
toggleConversationMode,
|
||||
prepareVoice,
|
||||
isMobile,
|
||||
voiceProvider,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* useMessageTTS Hook
|
||||
*
|
||||
* Hook for playing TTS on individual messages.
|
||||
* Uses the configured voice provider (browser, OpenAI, or macOS Say).
|
||||
*/
|
||||
|
||||
import { useCallback, useState } from 'react';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useServerTTS } from './useServerTTS';
|
||||
import { useSayTTS } from './useSayTTS';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { summarizeText, shouldSummarize, sanitizeForTTS } from '@/lib/voice/summarize';
|
||||
|
||||
export interface UseMessageTTSReturn {
|
||||
/** Whether TTS is currently playing for this message */
|
||||
isPlaying: boolean;
|
||||
/** Play the message text */
|
||||
play: (text: string) => Promise<void>;
|
||||
/** Stop playback */
|
||||
stop: () => void;
|
||||
}
|
||||
|
||||
export function useMessageTTS(): UseMessageTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
|
||||
const {
|
||||
voiceProvider,
|
||||
speechRate,
|
||||
speechPitch,
|
||||
speechVolume,
|
||||
sayVoice,
|
||||
browserVoice,
|
||||
openaiVoice,
|
||||
summarizeMessageTTS,
|
||||
summarizeCharacterThreshold,
|
||||
} = useConfigStore();
|
||||
|
||||
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable } = useServerTTS();
|
||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS();
|
||||
|
||||
const stop = useCallback(() => {
|
||||
setIsPlaying(false);
|
||||
stopServerTTS();
|
||||
stopSayTTS();
|
||||
browserVoiceService.cancelSpeech();
|
||||
}, [stopServerTTS, stopSayTTS]);
|
||||
|
||||
const play = useCallback(async (text: string) => {
|
||||
if (!text.trim()) return;
|
||||
|
||||
// Stop any existing playback
|
||||
stop();
|
||||
|
||||
setIsPlaying(true);
|
||||
|
||||
try {
|
||||
// Summarize text if enabled and over threshold
|
||||
let textToSpeak = text;
|
||||
if (summarizeMessageTTS && shouldSummarize(text, 'message')) {
|
||||
textToSpeak = await summarizeText(text, {
|
||||
threshold: summarizeCharacterThreshold,
|
||||
});
|
||||
} else {
|
||||
// Still sanitize for TTS even when not summarizing
|
||||
textToSpeak = sanitizeForTTS(text);
|
||||
}
|
||||
|
||||
if (voiceProvider === 'openai' && isServerTTSAvailable) {
|
||||
await speakServerTTS(textToSpeak, {
|
||||
voice: openaiVoice,
|
||||
speed: speechRate,
|
||||
summarize: false, // We already summarized client-side
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||
await speakSayTTS(textToSpeak, {
|
||||
voice: sayVoice,
|
||||
rate: wordsPerMinute,
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else {
|
||||
// Browser TTS
|
||||
await browserVoiceService.waitForVoices();
|
||||
await browserVoiceService.resumeAudioContext();
|
||||
await browserVoiceService.speakText(
|
||||
textToSpeak,
|
||||
navigator.language || 'en-US',
|
||||
() => setIsPlaying(false),
|
||||
{
|
||||
rate: speechRate,
|
||||
pitch: speechPitch,
|
||||
volume: speechVolume,
|
||||
voiceName: browserVoice || undefined,
|
||||
}
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[useMessageTTS] Playback error:', err);
|
||||
setIsPlaying(false);
|
||||
}
|
||||
}, [
|
||||
voiceProvider,
|
||||
speechRate,
|
||||
speechPitch,
|
||||
speechVolume,
|
||||
sayVoice,
|
||||
browserVoice,
|
||||
openaiVoice,
|
||||
summarizeMessageTTS,
|
||||
summarizeCharacterThreshold,
|
||||
isServerTTSAvailable,
|
||||
isSayTTSAvailable,
|
||||
speakServerTTS,
|
||||
speakSayTTS,
|
||||
stop,
|
||||
]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
play,
|
||||
stop,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
/**
|
||||
* useSayTTS Hook
|
||||
*
|
||||
* React hook for macOS 'say' command text-to-speech playback.
|
||||
* Uses the native macOS speech synthesis via server API.
|
||||
* Uses Web Audio API for playback (better iOS Safari support).
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const { speak, isPlaying, stop, isAvailable } = useSayTTS();
|
||||
*
|
||||
* // Speak text
|
||||
* await speak('Hello, this is a test');
|
||||
*
|
||||
* // Stop playback
|
||||
* stop();
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
export interface UseSayTTSReturn {
|
||||
/** Whether TTS is currently playing */
|
||||
isPlaying: boolean;
|
||||
/** Whether the macOS say command is available */
|
||||
isAvailable: boolean;
|
||||
/** Available voices */
|
||||
voices: Array<{ name: string; locale: string }>;
|
||||
/** Current error if any */
|
||||
error: string | null;
|
||||
/** Speak the given text */
|
||||
speak: (text: string, options?: SpeakOptions) => Promise<void>;
|
||||
/** Stop current playback */
|
||||
stop: () => void;
|
||||
/** Check if service is available */
|
||||
checkAvailability: () => Promise<boolean>;
|
||||
/** Unlock audio for mobile Safari - call this on user gesture */
|
||||
unlockAudio: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface SpeakOptions {
|
||||
/** Voice to use (defaults to Samantha) */
|
||||
voice?: string;
|
||||
/** Speech rate in words per minute (defaults to 200) */
|
||||
rate?: number;
|
||||
/** Callback when playback starts */
|
||||
onStart?: () => void;
|
||||
/** Callback when playback ends */
|
||||
onEnd?: () => void;
|
||||
/** Callback on error */
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
// Shared AudioContext for Web Audio API playback (better iOS support)
|
||||
let sharedAudioContext: AudioContext | null = null;
|
||||
|
||||
function getAudioContext(): AudioContext {
|
||||
if (!sharedAudioContext) {
|
||||
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
return sharedAudioContext;
|
||||
}
|
||||
|
||||
export function useSayTTS(): UseSayTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isAvailable, setIsAvailable] = useState(false);
|
||||
const [voices, setVoices] = useState<Array<{ name: string; locale: string }>>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
|
||||
// Unlock audio for mobile Safari - must be called within user gesture
|
||||
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||
try {
|
||||
// Get or create AudioContext
|
||||
const ctx = getAudioContext();
|
||||
|
||||
// Resume if suspended (required for iOS Safari)
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useSayTTS] AudioContext resumed');
|
||||
}
|
||||
|
||||
// Play a tiny silent buffer to fully unlock
|
||||
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(ctx.destination);
|
||||
source.start(0);
|
||||
|
||||
console.log('[useSayTTS] Audio unlocked for mobile playback');
|
||||
} catch (err) {
|
||||
console.error('[useSayTTS] Failed to unlock audio:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Check if macOS say is available
|
||||
const checkAvailability = useCallback(async (): Promise<boolean> => {
|
||||
try {
|
||||
const response = await fetch('/api/tts/say/status');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setIsAvailable(data.available);
|
||||
if (data.voices) {
|
||||
setVoices(data.voices);
|
||||
}
|
||||
return data.available;
|
||||
}
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
} catch (err) {
|
||||
console.error('[useSayTTS] Failed to check availability:', err);
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Check availability on mount
|
||||
useEffect(() => {
|
||||
checkAvailability();
|
||||
}, [checkAvailability]);
|
||||
|
||||
// Stop current playback
|
||||
const stop = useCallback(() => {
|
||||
if (audioSourceRef.current) {
|
||||
try {
|
||||
audioSourceRef.current.stop();
|
||||
} catch {
|
||||
// Already stopped
|
||||
}
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
}, []);
|
||||
|
||||
// Speak text using macOS say
|
||||
const speak = useCallback(async (text: string, options?: SpeakOptions): Promise<void> => {
|
||||
// Stop any existing playback
|
||||
stop();
|
||||
|
||||
if (!text.trim()) {
|
||||
setError('No text to speak');
|
||||
options?.onError?.('No text to speak');
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Create abort controller for this request
|
||||
abortControllerRef.current = new AbortController();
|
||||
|
||||
// Fetch audio from server
|
||||
const response = await fetch('/api/tts/say/speak', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text: text.trim(),
|
||||
voice: options?.voice || 'Samantha',
|
||||
rate: options?.rate || 200,
|
||||
}),
|
||||
signal: abortControllerRef.current.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
// Get audio data from response
|
||||
const audioBlob = await response.blob();
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
console.log('[useSayTTS] Received audio:', audioBlob.size, 'bytes');
|
||||
|
||||
// Use Web Audio API for playback (same as useServerTTS)
|
||||
const ctx = getAudioContext();
|
||||
|
||||
// Resume context if suspended
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useSayTTS] AudioContext resumed before playback');
|
||||
}
|
||||
|
||||
// Decode audio data
|
||||
console.log('[useSayTTS] Decoding audio data...');
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
|
||||
// Create source node
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
audioSourceRef.current = source;
|
||||
|
||||
// Set up event handlers
|
||||
source.onended = () => {
|
||||
console.log('[useSayTTS] Audio playback ended');
|
||||
setIsPlaying(false);
|
||||
audioSourceRef.current = null;
|
||||
options?.onEnd?.();
|
||||
};
|
||||
|
||||
// Start playback
|
||||
console.log('[useSayTTS] Starting audio playback via Web Audio API...');
|
||||
setIsPlaying(true);
|
||||
options?.onStart?.();
|
||||
source.start(0);
|
||||
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||
console.error('[useSayTTS] Error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
options?.onError?.(errorMsg);
|
||||
setIsPlaying(false);
|
||||
}
|
||||
}, [stop]);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
isAvailable,
|
||||
voices,
|
||||
error,
|
||||
speak,
|
||||
stop,
|
||||
checkAvailability,
|
||||
unlockAudio,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* useServerTTS Hook
|
||||
*
|
||||
* React hook for server-side text-to-speech playback.
|
||||
* Fetches audio from the server and plays it, bypassing mobile Safari restrictions.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const { speak, isPlaying, stop, isAvailable } = useServerTTS();
|
||||
*
|
||||
* // Speak text
|
||||
* await speak('Hello, this is a test');
|
||||
*
|
||||
* // Stop playback
|
||||
* stop();
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
export interface UseServerTTSReturn {
|
||||
/** Whether TTS is currently playing */
|
||||
isPlaying: boolean;
|
||||
/** Whether the server TTS service is available */
|
||||
isAvailable: boolean;
|
||||
/** Current error if any */
|
||||
error: string | null;
|
||||
/** Speak the given text */
|
||||
speak: (text: string, options?: SpeakOptions) => Promise<void>;
|
||||
/** Stop current playback */
|
||||
stop: () => void;
|
||||
/** Check if service is available */
|
||||
checkAvailability: () => Promise<boolean>;
|
||||
/** Unlock audio for mobile Safari - call this on user gesture before speaking */
|
||||
unlockAudio: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface SpeakOptions {
|
||||
/** Voice to use (defaults to coral) */
|
||||
voice?: string;
|
||||
/** Speech speed (0.25 to 4.0, defaults to 1.0) */
|
||||
speed?: number;
|
||||
/** Optional instructions for the voice */
|
||||
instructions?: string;
|
||||
/** Summarize long text before speaking (defaults to true) */
|
||||
summarize?: boolean;
|
||||
/** Provider ID for summarization model */
|
||||
providerId?: string;
|
||||
/** Model ID for summarization */
|
||||
modelId?: string;
|
||||
/** Character threshold for summarization (defaults to 200) */
|
||||
threshold?: number;
|
||||
/** Callback when playback starts */
|
||||
onStart?: () => void;
|
||||
/** Callback when playback ends */
|
||||
onEnd?: () => void;
|
||||
/** Callback on error */
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
// Shared AudioContext for Web Audio API playback (better iOS support)
|
||||
let sharedAudioContext: AudioContext | null = null;
|
||||
|
||||
function getAudioContext(): AudioContext {
|
||||
if (!sharedAudioContext) {
|
||||
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
return sharedAudioContext;
|
||||
}
|
||||
|
||||
export function useServerTTS(): UseServerTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isAvailable, setIsAvailable] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
|
||||
// Get current model, threshold, and max length from config store for summarization
|
||||
const { currentProviderId, currentModelId, summarizeCharacterThreshold, summarizeMaxLength, openaiApiKey } = useConfigStore();
|
||||
|
||||
// Check if server TTS is available
|
||||
const checkAvailability = useCallback(async (): Promise<boolean> => {
|
||||
try {
|
||||
const response = await fetch('/api/tts/status');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
// Available if server has key OR user has provided their own key
|
||||
const hasServerKey = data.available;
|
||||
const hasClientKey = openaiApiKey && openaiApiKey.trim().length > 0;
|
||||
const available = hasServerKey || hasClientKey;
|
||||
setIsAvailable(available);
|
||||
return available;
|
||||
}
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
} catch (err) {
|
||||
console.error('[useServerTTS] Failed to check availability:', err);
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
}
|
||||
}, [openaiApiKey]);
|
||||
|
||||
// Check availability on mount and when API key changes
|
||||
useEffect(() => {
|
||||
checkAvailability();
|
||||
}, [checkAvailability]);
|
||||
|
||||
// Stop current playback
|
||||
const stop = useCallback(() => {
|
||||
// Stop Web Audio API source
|
||||
if (audioSourceRef.current) {
|
||||
try {
|
||||
audioSourceRef.current.stop();
|
||||
} catch {
|
||||
// Already stopped
|
||||
}
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
}, []);
|
||||
|
||||
// Pre-unlock audio for mobile Safari
|
||||
// This must be called within a user gesture context
|
||||
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||
try {
|
||||
// Get or create AudioContext
|
||||
const ctx = getAudioContext();
|
||||
|
||||
// Resume if suspended (required for iOS Safari)
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useServerTTS] AudioContext resumed');
|
||||
}
|
||||
|
||||
// Play a tiny silent buffer to fully unlock
|
||||
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(ctx.destination);
|
||||
source.start(0);
|
||||
|
||||
console.log('[useServerTTS] Audio unlocked for mobile playback');
|
||||
} catch (err) {
|
||||
console.error('[useServerTTS] Failed to unlock audio:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Speak text using server TTS
|
||||
const speak = useCallback(async (text: string, options?: SpeakOptions): Promise<void> => {
|
||||
// Stop any existing playback
|
||||
stop();
|
||||
|
||||
if (!text.trim()) {
|
||||
setError('No text to speak');
|
||||
options?.onError?.('No text to speak');
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Unlock audio context first (required for mobile Safari)
|
||||
// Must be done before any async operations to stay within user gesture context
|
||||
const ctx = getAudioContext();
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useServerTTS] AudioContext resumed');
|
||||
}
|
||||
|
||||
// Play a silent buffer to fully unlock audio on iOS
|
||||
const silentBuffer = ctx.createBuffer(1, 1, 22050);
|
||||
const silentSource = ctx.createBufferSource();
|
||||
silentSource.buffer = silentBuffer;
|
||||
silentSource.connect(ctx.destination);
|
||||
silentSource.start(0);
|
||||
|
||||
// Create abort controller for this request
|
||||
abortControllerRef.current = new AbortController();
|
||||
|
||||
const voice = options?.voice || 'nova';
|
||||
console.log('[useServerTTS] Speaking with voice:', voice, 'options:', options);
|
||||
|
||||
// Fetch audio from server
|
||||
const response = await fetch('/api/tts/speak', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text: text.trim(),
|
||||
voice,
|
||||
speed: options?.speed || 0.9,
|
||||
instructions: options?.instructions,
|
||||
summarize: options?.summarize ?? true, // Summarize by default for voice output
|
||||
// Use provided provider/model, or fall back to current chat model
|
||||
providerId: options?.providerId || currentProviderId || undefined,
|
||||
modelId: options?.modelId || currentModelId || undefined,
|
||||
// Use provided threshold, or fall back to user setting, or default to 200
|
||||
threshold: options?.threshold ?? summarizeCharacterThreshold ?? 200,
|
||||
// Max character length for summaries
|
||||
maxLength: summarizeMaxLength ?? 500,
|
||||
// Send API key from settings if available
|
||||
apiKey: openaiApiKey || undefined,
|
||||
}),
|
||||
signal: abortControllerRef.current.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
// Get audio data from response
|
||||
const audioBlob = await response.blob();
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
|
||||
// Decode audio data using the same context we unlocked earlier
|
||||
console.log('[useServerTTS] Decoding audio data...');
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
|
||||
// Create source node
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
audioSourceRef.current = source;
|
||||
|
||||
// Set up event handlers
|
||||
source.onended = () => {
|
||||
console.log('[useServerTTS] Audio playback ended');
|
||||
setIsPlaying(false);
|
||||
audioSourceRef.current = null;
|
||||
options?.onEnd?.();
|
||||
};
|
||||
|
||||
// Start playback
|
||||
console.log('[useServerTTS] Starting audio playback via Web Audio API...');
|
||||
setIsPlaying(true);
|
||||
options?.onStart?.();
|
||||
source.start(0);
|
||||
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError') {
|
||||
// Request was aborted, don't show error
|
||||
return;
|
||||
}
|
||||
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||
console.error('[useServerTTS] Error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
options?.onError?.(errorMsg);
|
||||
setIsPlaying(false);
|
||||
}
|
||||
}, [stop, currentProviderId, currentModelId, summarizeCharacterThreshold, summarizeMaxLength, openaiApiKey]);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
isAvailable,
|
||||
error,
|
||||
speak,
|
||||
stop,
|
||||
checkAvailability,
|
||||
unlockAudio,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useSessionStore } from '@/stores/useSessionStore';
|
||||
import { voiceHooks, isVoiceSessionStarted } from '@/lib/voice';
|
||||
|
||||
/**
|
||||
* Hook that syncs session events (messages, permissions) to the voice agent.
|
||||
* Call this inside VoiceProvider to enable session awareness during voice.
|
||||
*/
|
||||
export function useVoiceContext() {
|
||||
const currentSessionId = useSessionStore((s) => s.currentSessionId);
|
||||
const messages = useSessionStore((s) =>
|
||||
currentSessionId ? s.messages.get(currentSessionId) : undefined
|
||||
);
|
||||
const permissions = useSessionStore((s) =>
|
||||
currentSessionId ? s.permissions.get(currentSessionId) : undefined
|
||||
);
|
||||
|
||||
// Track last seen message count to only forward new messages
|
||||
const lastMessageCountRef = useRef(0);
|
||||
|
||||
// Forward new messages to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !messages || !isVoiceSessionStarted()) return;
|
||||
|
||||
const currentCount = messages.length;
|
||||
if (currentCount <= lastMessageCountRef.current) return;
|
||||
|
||||
// Get only new messages (messages since last check)
|
||||
const newMessages = messages.slice(lastMessageCountRef.current);
|
||||
lastMessageCountRef.current = currentCount;
|
||||
|
||||
// Format for voice hooks (extract role and content)
|
||||
const formattedMessages = newMessages.map(m => ({
|
||||
role: m.info.role,
|
||||
content: m.parts.map(p => ('text' in p ? p.text : '')).join('')
|
||||
}));
|
||||
|
||||
voiceHooks.onMessages(currentSessionId, formattedMessages);
|
||||
}, [currentSessionId, messages]);
|
||||
|
||||
// Forward permission requests to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !permissions || permissions.length === 0) return;
|
||||
if (!isVoiceSessionStarted()) return;
|
||||
|
||||
const request = permissions[0];
|
||||
if (!request) return;
|
||||
|
||||
voiceHooks.onPermissionRequested(
|
||||
currentSessionId,
|
||||
request.id,
|
||||
request.permission,
|
||||
request.metadata
|
||||
);
|
||||
}, [currentSessionId, permissions]);
|
||||
|
||||
// Reset message count when session changes
|
||||
useEffect(() => {
|
||||
lastMessageCountRef.current = 0;
|
||||
}, [currentSessionId]);
|
||||
}
|
||||
Reference in New Issue
Block a user