feat(voice): add voice input/output support with multiple providers (#281)
* feat(voice): add voice input/output support with multiple providers - Add BrowserVoiceButton component for Web Speech API voice input - Add VoiceProvider context for managing voice state across the app - Add TTS (Text-to-Speech) support with browser, macOS Say, and OpenAI providers - Add message TTS buttons to read assistant messages aloud - Add VoiceSettings page in OpenChamber settings - Add server endpoints for TTS and summarization services - Include slider component for voice rate/pitch/volume controls - Add hidden session support for background voice operations - Add Caddyfile for HTTPS support (required for microphone access) * fix: Build errors fixed and removed outdated ElevenLabs test code. * refactor(voice): use zen API with gpt-5-nano for TTS summarization Replace the hidden session + OpenCode SDK approach with direct calls to the opencode.ai zen API (same pattern used for commit message and PR description generation). - Rewrite summarization-service.js to call zen/v1/responses with gpt-5-nano - Remove hidden session logic (hiddenSession.ts, sessionStore filtering) - Remove summarizeModel setting and model selector from VoiceSettings - Simplify client-side summarize.ts to no longer pass model params - Clean up callers in useMessageTTS and useBrowserVoice * fix(voice): remove false 'voice not supported' warning in settings Mobile Safari does support voice but the isSupported check was incorrectly flagging it. Remove the warning banner entirely. * feat(voice): add configurable summary length limit for TTS output Add a slider (50-2000 chars) in voice settings to control max summary length. The limit is passed through the summarize endpoint and speak endpoint to the zen API prompt, with token budget scaled accordingly. * fix(voice): add diagnostic logging and sanitize TTS fallback Add console logging throughout the summarization flow (client + server) to trace why text may not be summarized. Fix silent error swallowing in /api/tts/speak. Always apply sanitizeForTTS even when summarization is disabled so raw markdown/code is never spoken verbatim. * fix(voice): fix token budget starving model of output tokens max_output_tokens includes both reasoning and output tokens. With effort:'low', reasoning alone consumes ~128 tokens, so a budget of 100 left zero tokens for the actual summary text. Use a fixed 1000 token budget (matching commit message generation) and control output length via the prompt's character limit instruction instead. * chore(voice): remove diagnostic logging from summarization flow * fix(voice): don't request mic permission on mobile page load Remove the useEffect that pre-requested microphone permission when the BrowserVoiceButton component mounted on mobile. This caused an unwanted permission prompt immediately on page load before the user tapped the mic icon. Permission is now only requested on explicit user interaction. * fix(voice): remove unused BrowserVoiceButton binding * fix(voice): desktop mic flow + non-continuous draft mode * fix(voice): stabilize continuous loop and polish controls * feat(settings): mark voice section experimental --------- Co-authored-by: Bohdan Triapitsyn <artmore@protonmail.com>
This commit is contained in:
committed by
GitHub
co-authored by
Bohdan Triapitsyn
parent
6776ac31c2
commit
1ed5316ac7
@@ -0,0 +1,22 @@
|
||||
{
|
||||
auto_https off
|
||||
|
||||
}
|
||||
|
||||
:3443 {
|
||||
bind 0.0.0.0
|
||||
tls /tmp/localhost.crt /tmp/localhost.key
|
||||
reverse_proxy [::1]:3001 {
|
||||
transport http {
|
||||
read_timeout 120s
|
||||
write_timeout 120s
|
||||
compression off
|
||||
}
|
||||
header_up Host {host}
|
||||
header_up X-Real-IP {remote_host}
|
||||
header_up X-Forwarded-For {remote_host}
|
||||
header_up X-Forwarded-Proto {scheme}
|
||||
header_down -Transfer-Encoding
|
||||
flush_interval -1
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -101,7 +101,6 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^1.1.1",
|
||||
"codemirror-lang-elixir": "^4.0.0",
|
||||
"electron-context-menu": "^4.1.1",
|
||||
"electron-store": "^11.0.2",
|
||||
"express": "^5.1.0",
|
||||
@@ -118,11 +117,13 @@
|
||||
"sonner": "^2.0.7",
|
||||
"tailwind-merge": "^3.3.1",
|
||||
"yaml": "^2.8.1",
|
||||
"zod": "^4.3.6",
|
||||
"zustand": "^5.0.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.33.0",
|
||||
"@tailwindcss/postcss": "^4.0.0",
|
||||
"@types/dom-speech-recognition": "^0.0.7",
|
||||
"@types/node": "^24.3.1",
|
||||
"@types/react": "^19.1.10",
|
||||
"@types/react-dom": "^19.1.7",
|
||||
|
||||
@@ -14,5 +14,9 @@
|
||||
<string>OpenChamber needs access to work with your projects.</string>
|
||||
<key>NSDownloadsFolderUsageDescription</key>
|
||||
<string>OpenChamber needs access to work with your projects.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>OpenChamber needs microphone access for voice input.</string>
|
||||
<key>NSSpeechRecognitionUsageDescription</key>
|
||||
<string>OpenChamber needs speech recognition to transcribe voice input.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
|
||||
@@ -73,6 +73,7 @@
|
||||
"strip-json-comments": "^5.0.3",
|
||||
"tailwind-merge": "^3.3.1",
|
||||
"yaml": "^2.8.1",
|
||||
"zod": "^4.3.6",
|
||||
"zustand": "^5.0.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -28,6 +28,7 @@ import { ConfigUpdateOverlay } from '@/components/ui/ConfigUpdateOverlay';
|
||||
import { AboutDialog } from '@/components/ui/AboutDialog';
|
||||
import { RuntimeAPIProvider } from '@/contexts/RuntimeAPIProvider';
|
||||
import { registerRuntimeAPIs } from '@/contexts/runtimeAPIRegistry';
|
||||
import { VoiceProvider } from '@/components/voice';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
import { useGitHubAuthStore } from '@/stores/useGitHubAuthStore';
|
||||
import type { RuntimeAPIs } from '@/lib/api/types';
|
||||
@@ -305,6 +306,7 @@ function App({ apis }: AppProps) {
|
||||
<RuntimeAPIProvider apis={apis}>
|
||||
<GitPollingProvider>
|
||||
<FireworksProvider>
|
||||
<VoiceProvider>
|
||||
<div className="h-full text-foreground bg-background">
|
||||
<MainLayout />
|
||||
<Toaster />
|
||||
@@ -314,6 +316,7 @@ function App({ apis }: AppProps) {
|
||||
<MemoryDebugPanel onClose={() => setShowMemoryDebug(false)} />
|
||||
)}
|
||||
</div>
|
||||
</VoiceProvider>
|
||||
</FireworksProvider>
|
||||
</GitPollingProvider>
|
||||
</RuntimeAPIProvider>
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
RiFileUploadLine,
|
||||
RiSendPlane2Line,
|
||||
} from '@remixicon/react';
|
||||
import { BrowserVoiceButton } from '@/components/voice';
|
||||
import { useSessionStore } from '@/stores/useSessionStore';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
@@ -1438,6 +1439,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
const iconSizeClass = isMobile ? 'h-[18px] w-[18px]' : (isVSCode ? 'h-4 w-4' : 'h-[18px] w-[18px]');
|
||||
|
||||
const iconButtonBaseClass = 'flex items-center justify-center text-muted-foreground transition-none outline-none focus:outline-none flex-shrink-0';
|
||||
const footerIconButtonClass = cn(iconButtonBaseClass, buttonSizeClass);
|
||||
|
||||
// Send button - respects queue mode setting
|
||||
const sendButton = (
|
||||
@@ -1472,8 +1474,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
handlePrimaryAction();
|
||||
}}
|
||||
className={cn(
|
||||
iconButtonBaseClass,
|
||||
buttonSizeClass,
|
||||
footerIconButtonClass,
|
||||
canSend && (currentSessionId || newSessionDraftOpen)
|
||||
? 'text-primary hover:text-primary'
|
||||
: 'opacity-30'
|
||||
@@ -1514,8 +1515,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
handleQueueMessage();
|
||||
}}
|
||||
className={cn(
|
||||
iconButtonBaseClass,
|
||||
buttonSizeClass,
|
||||
footerIconButtonClass,
|
||||
'absolute bottom-full left-1/2 -translate-x-1/2 mb-1',
|
||||
hasContent && currentSessionId
|
||||
? 'text-primary hover:text-primary'
|
||||
@@ -1533,8 +1533,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
type="button"
|
||||
onClick={handleAbort}
|
||||
className={cn(
|
||||
iconButtonBaseClass,
|
||||
buttonSizeClass,
|
||||
footerIconButtonClass,
|
||||
'text-[var(--status-error)] hover:text-[var(--status-error)]'
|
||||
)}
|
||||
aria-label="Stop generating"
|
||||
@@ -1586,7 +1585,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(iconButtonBaseClass, isMobile && 'h-7 w-7')}
|
||||
className={footerIconButtonClass}
|
||||
title="Add attachment"
|
||||
aria-label="Add attachment"
|
||||
>
|
||||
@@ -1622,7 +1621,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
<button
|
||||
type='button'
|
||||
onClick={onOpenSettings}
|
||||
className={cn(iconButtonBaseClass, isMobile && 'h-7 w-7')}
|
||||
className={footerIconButtonClass}
|
||||
title='Model and agent settings'
|
||||
aria-label='Model and agent settings'
|
||||
>
|
||||
@@ -1631,13 +1630,13 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
) : null;
|
||||
|
||||
const attachmentsControls = (
|
||||
<div className="flex items-center gap-x-1">
|
||||
<div className="flex items-center gap-x-1.5">
|
||||
{isMobile ? (
|
||||
<button
|
||||
type="button"
|
||||
className={cn(
|
||||
iconButtonBaseClass,
|
||||
'h-7 w-7 rounded-md text-muted-foreground',
|
||||
footerIconButtonClass,
|
||||
'rounded-md text-muted-foreground',
|
||||
'hover:bg-interactive-hover/40 hover:text-foreground'
|
||||
)}
|
||||
onPointerDownCapture={(event) => {
|
||||
@@ -1873,6 +1872,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
<MobileAgentButton onCycleAgent={handleCycleAgent} onOpenAgentPanel={() => setMobileControlsPanel('agent')} className="min-w-0 flex-shrink" />
|
||||
</div>
|
||||
<div className="flex items-center gap-x-1 flex-shrink-0">
|
||||
<BrowserVoiceButton />
|
||||
{actionButtons}
|
||||
</div>
|
||||
</div>
|
||||
@@ -1898,6 +1898,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ onOpenSettings, scrollToBo
|
||||
</div>
|
||||
<div className={cn('flex items-center flex-1 justify-end', footerGapClass, 'md:gap-x-3')}>
|
||||
<ModelControls className={cn('flex-1 min-w-0 justify-end')} />
|
||||
<BrowserVoiceButton />
|
||||
{actionButtons}
|
||||
</div>
|
||||
</>
|
||||
|
||||
@@ -14,7 +14,7 @@ import { isEmptyTextPart, extractTextContent } from './partUtils';
|
||||
import { FadeInOnReveal } from './FadeInOnReveal';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import { RiCheckLine, RiFileCopyLine, RiChatNewLine, RiArrowGoBackLine, RiGitBranchLine, RiHourglassLine } from '@remixicon/react';
|
||||
import { RiCheckLine, RiFileCopyLine, RiChatNewLine, RiArrowGoBackLine, RiGitBranchLine, RiHourglassLine, RiVolumeUpLine, RiStopLine } from '@remixicon/react';
|
||||
import { ArrowsMerge } from '@/components/icons/ArrowsMerge';
|
||||
import type { ContentChangeReason } from '@/hooks/useChatScrollManager';
|
||||
|
||||
@@ -24,6 +24,8 @@ import { useSessionStore } from '@/stores/useSessionStore';
|
||||
import { useUIStore } from '@/stores/useUIStore';
|
||||
import { flattenAssistantTextParts } from '@/lib/messages/messageText';
|
||||
import { MULTIRUN_EXECUTION_FORK_PROMPT_META_TEXT } from '@/lib/messages/executionMeta';
|
||||
import { useMessageTTS } from '@/hooks/useMessageTTS';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { TextSelectionMenu } from './TextSelectionMenu';
|
||||
|
||||
const formatTurnDuration = (durationMs: number): string => {
|
||||
@@ -318,6 +320,19 @@ const AssistantMessageBody: React.FC<Omit<MessageBodyProps, 'isUser'>> = ({
|
||||
const isLastAssistantInTurn = turnGroupingContext?.isLastAssistantInTurn ?? false;
|
||||
const hasStopFinish = messageFinish === 'stop';
|
||||
|
||||
// TTS for message playback
|
||||
const { isPlaying: isTTSPlaying, play: playTTS, stop: stopTTS } = useMessageTTS();
|
||||
const showMessageTTSButtons = useConfigStore((state) => state.showMessageTTSButtons);
|
||||
const voiceProvider = useConfigStore((state) => state.voiceProvider);
|
||||
|
||||
const readAloudTooltip = React.useMemo(() => {
|
||||
if (isTTSPlaying) {
|
||||
return 'Stop speaking';
|
||||
}
|
||||
const providerLabel = voiceProvider === 'browser' ? 'Browser' : voiceProvider === 'openai' ? 'OpenAI' : 'Say';
|
||||
return `Read aloud (${providerLabel} voice)`;
|
||||
}, [isTTSPlaying, voiceProvider]);
|
||||
|
||||
|
||||
const hasTools = toolParts.length > 0;
|
||||
|
||||
@@ -503,6 +518,24 @@ const AssistantMessageBody: React.FC<Omit<MessageBodyProps, 'isUser'>> = ({
|
||||
[assistantTextParts, openMultiRunLauncherWithPrompt]
|
||||
);
|
||||
|
||||
const handleTTSClick = React.useCallback(
|
||||
(event: React.MouseEvent<HTMLButtonElement>) => {
|
||||
event.stopPropagation();
|
||||
event.preventDefault();
|
||||
|
||||
if (isTTSPlaying) {
|
||||
stopTTS();
|
||||
return;
|
||||
}
|
||||
|
||||
const messageText = flattenAssistantTextParts(assistantTextParts);
|
||||
if (messageText.trim()) {
|
||||
void playTTS(messageText);
|
||||
}
|
||||
},
|
||||
[assistantTextParts, isTTSPlaying, playTTS, stopTTS]
|
||||
);
|
||||
|
||||
React.useEffect(() => {
|
||||
return () => {
|
||||
clearCopyHintTimeout();
|
||||
@@ -882,6 +915,31 @@ const AssistantMessageBody: React.FC<Omit<MessageBodyProps, 'isUser'>> = ({
|
||||
<TooltipContent sideOffset={6}>Start new multi-run from this answer</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
{showMessageTTSButtons && hasCopyableText && (
|
||||
<Tooltip delayDuration={1000}>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className={cn(
|
||||
'h-8 w-8 bg-transparent hover:!bg-transparent active:!bg-transparent focus-visible:!bg-transparent focus-visible:ring-2 focus-visible:ring-primary/50',
|
||||
isTTSPlaying ? 'text-green-500' : 'text-muted-foreground hover:text-foreground'
|
||||
)}
|
||||
aria-label={isTTSPlaying ? 'Stop speaking' : 'Read aloud'}
|
||||
onPointerDown={(event) => event.stopPropagation()}
|
||||
onClick={handleTTSClick}
|
||||
>
|
||||
{isTTSPlaying ? (
|
||||
<RiStopLine className="h-3.5 w-3.5" />
|
||||
) : (
|
||||
<RiVolumeUpLine className="h-3.5 w-3.5" />
|
||||
)}
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent sideOffset={6}>{readAloudTooltip}</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
{onCopyMessage && (
|
||||
<Tooltip delayDuration={1000}>
|
||||
<TooltipTrigger asChild>
|
||||
|
||||
@@ -26,6 +26,7 @@ interface ModelSelectorProps {
|
||||
onChange: (providerId: string, modelId: string) => void;
|
||||
className?: string;
|
||||
allowedProviderIds?: string[];
|
||||
placeholder?: string;
|
||||
}
|
||||
|
||||
const COMPACT_NUMBER_FORMATTER = new Intl.NumberFormat('en-US', {
|
||||
@@ -51,7 +52,8 @@ export const ModelSelector: React.FC<ModelSelectorProps> = ({
|
||||
modelId,
|
||||
onChange,
|
||||
className,
|
||||
allowedProviderIds
|
||||
allowedProviderIds,
|
||||
placeholder
|
||||
}) => {
|
||||
const { providers, modelsMetadata } = useConfigStore();
|
||||
const isMobile = useUIStore(state => state.isMobile);
|
||||
@@ -460,7 +462,7 @@ export const ModelSelector: React.FC<ModelSelectorProps> = ({
|
||||
closeMobilePanel();
|
||||
}}
|
||||
>
|
||||
<span className="typography-meta text-muted-foreground">No model (optional)</span>
|
||||
<span className="typography-meta text-muted-foreground">{placeholder || 'No model (optional)'}</span>
|
||||
</button>
|
||||
</div>
|
||||
</MobileOverlayPanel>
|
||||
@@ -488,7 +490,7 @@ export const ModelSelector: React.FC<ModelSelectorProps> = ({
|
||||
<RiPencilAiLine className="h-3 w-3 text-muted-foreground" />
|
||||
)}
|
||||
<span className="typography-meta font-medium text-foreground">
|
||||
{providerId && modelId ? `${providerId}/${modelId}` : 'Select model...'}
|
||||
{providerId && modelId ? `${providerId}/${modelId}` : (placeholder || 'Select model...')}
|
||||
</span>
|
||||
</div>
|
||||
<RiArrowDownSLine className="h-3 w-3 text-muted-foreground" />
|
||||
@@ -512,7 +514,7 @@ export const ModelSelector: React.FC<ModelSelectorProps> = ({
|
||||
<RiPencilAiLine className="h-3 w-3 text-muted-foreground" />
|
||||
)}
|
||||
<span className="typography-micro font-medium whitespace-nowrap">
|
||||
{providerId && modelId ? `${providerId}/${modelId}` : 'Not selected'}
|
||||
{providerId && modelId ? `${providerId}/${modelId}` : (placeholder || 'Not selected')}
|
||||
</span>
|
||||
<RiArrowDownSLine className="h-3 w-3 flex-shrink-0 text-muted-foreground" />
|
||||
</div>
|
||||
@@ -601,7 +603,7 @@ export const ModelSelector: React.FC<ModelSelectorProps> = ({
|
||||
onClick={() => handleProviderAndModelChange('', '')}
|
||||
>
|
||||
<RiCloseLine className="h-3.5 w-3.5 text-muted-foreground" />
|
||||
<span className="text-muted-foreground">Not selected</span>
|
||||
<span className="text-muted-foreground">{placeholder || 'Not selected'}</span>
|
||||
{!providerId && !modelId && (
|
||||
<RiCheckLine className="h-4 w-4 text-primary ml-auto" />
|
||||
)}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { GitSettings } from './GitSettings';
|
||||
import { WorktreeSectionContent } from './WorktreeSectionContent';
|
||||
import { NotificationSettings } from './NotificationSettings';
|
||||
import { GitHubSettings } from './GitHubSettings';
|
||||
import { VoiceSettings } from './VoiceSettings';
|
||||
import { OpenCodeCliSettings } from './OpenCodeCliSettings';
|
||||
import { ScrollableOverlay } from '@/components/ui/ScrollableOverlay';
|
||||
import { useDeviceInfo } from '@/lib/device';
|
||||
@@ -70,6 +71,8 @@ export const OpenChamberPage: React.FC<OpenChamberPageProps> = ({ section }) =>
|
||||
return <GitHubSectionContent />;
|
||||
case 'notifications':
|
||||
return <NotificationSectionContent />;
|
||||
case 'voice':
|
||||
return <VoiceSectionContent />;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
@@ -149,3 +152,8 @@ const GitHubSectionContent: React.FC = () => {
|
||||
const NotificationSectionContent: React.FC = () => {
|
||||
return <NotificationSettings />;
|
||||
};
|
||||
|
||||
// Voice section: Language selection and continuous mode
|
||||
const VoiceSectionContent: React.FC = () => {
|
||||
return <VoiceSettings />;
|
||||
};
|
||||
|
||||
@@ -5,7 +5,7 @@ import { isVSCodeRuntime, isWebRuntime } from '@/lib/desktop';
|
||||
import { AboutSettings } from './AboutSettings';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export type OpenChamberSection = 'visual' | 'chat' | 'sessions' | 'git' | 'github' | 'notifications';
|
||||
export type OpenChamberSection = 'visual' | 'chat' | 'sessions' | 'git' | 'github' | 'notifications' | 'voice';
|
||||
|
||||
interface OpenChamberSidebarProps {
|
||||
selectedSection: OpenChamberSection;
|
||||
@@ -16,6 +16,7 @@ interface SectionGroup {
|
||||
id: OpenChamberSection;
|
||||
label: string;
|
||||
items: string[];
|
||||
badge?: string;
|
||||
webOnly?: boolean;
|
||||
hideInVSCode?: boolean;
|
||||
}
|
||||
@@ -53,6 +54,12 @@ const OPENCHAMBER_SECTION_GROUPS: SectionGroup[] = [
|
||||
label: 'Notifications',
|
||||
items: ['Native'],
|
||||
},
|
||||
{
|
||||
id: 'voice',
|
||||
label: 'Voice',
|
||||
items: ['Language', 'Continuous Mode'],
|
||||
badge: 'experimental',
|
||||
},
|
||||
];
|
||||
|
||||
export const OpenChamberSidebar: React.FC<OpenChamberSidebarProps> = ({
|
||||
@@ -95,9 +102,16 @@ export const OpenChamberSidebar: React.FC<OpenChamberSidebarProps> = ({
|
||||
onClick={() => onSelectSection(group.id)}
|
||||
className="w-full text-left flex flex-col gap-0 rounded-sm focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-primary/50"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="typography-ui-label font-normal text-foreground">
|
||||
{group.label}
|
||||
</span>
|
||||
{group.badge && (
|
||||
<span className="text-[10px] leading-none uppercase font-bold tracking-tight bg-[var(--status-warning-background)] text-[var(--status-warning)] border border-[var(--status-warning-border)] px-1.5 py-0.5 rounded">
|
||||
{group.badge}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="typography-micro text-muted-foreground/60 leading-tight">
|
||||
{group.items.join(' · ')}
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,874 @@
|
||||
import React, { useEffect, useState, useCallback, useMemo } from 'react';
|
||||
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { SettingsSection } from '@/components/sections/shared/SettingsSection';
|
||||
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Slider } from '@/components/ui/slider';
|
||||
import { RiMicLine, RiAlertLine, RiVolumeUpLine, RiSpeedLine, RiMusicLine, RiSoundModuleLine, RiAppleLine, RiPlayLine, RiStopLine, RiChromeLine, RiFileTextLine, RiKeyLine, RiCloseLine } from '@remixicon/react';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
|
||||
// Common language options with display names
|
||||
// Shared with BrowserVoiceButton.tsx
|
||||
const LANGUAGE_OPTIONS = [
|
||||
{ value: 'en-US', label: 'English' },
|
||||
{ value: 'es-ES', label: 'Español' },
|
||||
{ value: 'fr-FR', label: 'Français' },
|
||||
{ value: 'de-DE', label: 'Deutsch' },
|
||||
{ value: 'ja-JP', label: '日本語' },
|
||||
{ value: 'zh-CN', label: '中文' },
|
||||
{ value: 'pt-BR', label: 'Português' },
|
||||
{ value: 'it-IT', label: 'Italiano' },
|
||||
{ value: 'ko-KR', label: '한국어' },
|
||||
{ value: 'uk-UA', label: 'Українська' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Voice settings section for OpenChamber settings
|
||||
* Allows users to configure voice conversation preferences
|
||||
*/
|
||||
export const VoiceSettings: React.FC = () => {
|
||||
const {
|
||||
isSupported,
|
||||
language,
|
||||
setLanguage,
|
||||
} = useBrowserVoice();
|
||||
const {
|
||||
voiceProvider,
|
||||
setVoiceProvider,
|
||||
speechRate,
|
||||
setSpeechRate,
|
||||
speechPitch,
|
||||
setSpeechPitch,
|
||||
speechVolume,
|
||||
setSpeechVolume,
|
||||
sayVoice,
|
||||
setSayVoice,
|
||||
browserVoice,
|
||||
setBrowserVoice,
|
||||
openaiVoice,
|
||||
setOpenaiVoice,
|
||||
openaiApiKey,
|
||||
setOpenaiApiKey,
|
||||
showMessageTTSButtons,
|
||||
setShowMessageTTSButtons,
|
||||
voiceModeEnabled,
|
||||
setVoiceModeEnabled,
|
||||
summarizeMessageTTS,
|
||||
setSummarizeMessageTTS,
|
||||
summarizeVoiceConversation,
|
||||
setSummarizeVoiceConversation,
|
||||
summarizeCharacterThreshold,
|
||||
setSummarizeCharacterThreshold,
|
||||
summarizeMaxLength,
|
||||
setSummarizeMaxLength,
|
||||
} = useConfigStore();
|
||||
|
||||
// Check if macOS 'say' is available and get voices
|
||||
const [isSayAvailable, setIsSayAvailable] = useState(false);
|
||||
const [sayVoices, setSayVoices] = useState<Array<{ name: string; locale: string }>>([]);
|
||||
const [isPreviewPlaying, setIsPreviewPlaying] = useState(false);
|
||||
const [previewAudio, setPreviewAudio] = useState<HTMLAudioElement | null>(null);
|
||||
|
||||
// Check if OpenAI TTS is available
|
||||
const [isOpenAIAvailable, setIsOpenAIAvailable] = useState(false);
|
||||
const [isOpenAIPreviewPlaying, setIsOpenAIPreviewPlaying] = useState(false);
|
||||
const [openaiPreviewAudio, setOpenaiPreviewAudio] = useState<HTMLAudioElement | null>(null);
|
||||
|
||||
// Browser voices
|
||||
const [browserVoices, setBrowserVoices] = useState<SpeechSynthesisVoice[]>([]);
|
||||
const [isBrowserPreviewPlaying, setIsBrowserPreviewPlaying] = useState(false);
|
||||
|
||||
// Load browser voices
|
||||
useEffect(() => {
|
||||
const loadVoices = async () => {
|
||||
const voices = await browserVoiceService.waitForVoices();
|
||||
setBrowserVoices(voices);
|
||||
};
|
||||
loadVoices();
|
||||
|
||||
// Also listen for voice changes (Chrome loads voices asynchronously)
|
||||
if ('speechSynthesis' in window) {
|
||||
window.speechSynthesis.onvoiceschanged = () => {
|
||||
setBrowserVoices(window.speechSynthesis.getVoices());
|
||||
};
|
||||
}
|
||||
|
||||
return () => {
|
||||
if ('speechSynthesis' in window) {
|
||||
window.speechSynthesis.onvoiceschanged = null;
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Filter and sort browser voices by language
|
||||
const filteredBrowserVoices = useMemo(() => {
|
||||
// Group voices by language, prioritize English voices at top
|
||||
return browserVoices
|
||||
.filter(v => v.lang) // Only voices with a language
|
||||
.sort((a, b) => {
|
||||
// Prioritize English voices
|
||||
const aIsEnglish = a.lang.startsWith('en');
|
||||
const bIsEnglish = b.lang.startsWith('en');
|
||||
if (aIsEnglish && !bIsEnglish) return -1;
|
||||
if (!aIsEnglish && bIsEnglish) return 1;
|
||||
// Then sort by language, then by name
|
||||
const langCompare = a.lang.localeCompare(b.lang);
|
||||
if (langCompare !== 0) return langCompare;
|
||||
return a.name.localeCompare(b.name);
|
||||
});
|
||||
}, [browserVoices]);
|
||||
|
||||
// Preview browser voice
|
||||
const previewBrowserVoice = useCallback(() => {
|
||||
if (isBrowserPreviewPlaying) {
|
||||
browserVoiceService.cancelSpeech();
|
||||
setIsBrowserPreviewPlaying(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const selectedVoice = browserVoices.find(v => v.name === browserVoice);
|
||||
const voiceName = selectedVoice?.name ?? 'your browser voice';
|
||||
const previewText = `Hello! I'm ${voiceName}. This is how I sound.`;
|
||||
|
||||
setIsBrowserPreviewPlaying(true);
|
||||
|
||||
const utterance = new SpeechSynthesisUtterance(previewText);
|
||||
utterance.rate = speechRate;
|
||||
utterance.pitch = speechPitch;
|
||||
utterance.volume = speechVolume;
|
||||
|
||||
if (selectedVoice) {
|
||||
utterance.voice = selectedVoice;
|
||||
utterance.lang = selectedVoice.lang;
|
||||
}
|
||||
|
||||
utterance.onend = () => setIsBrowserPreviewPlaying(false);
|
||||
utterance.onerror = () => setIsBrowserPreviewPlaying(false);
|
||||
|
||||
window.speechSynthesis.cancel();
|
||||
window.speechSynthesis.speak(utterance);
|
||||
}, [browserVoice, browserVoices, speechRate, speechPitch, speechVolume, isBrowserPreviewPlaying]);
|
||||
|
||||
// Cleanup browser preview on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (isBrowserPreviewPlaying) {
|
||||
browserVoiceService.cancelSpeech();
|
||||
}
|
||||
};
|
||||
}, [isBrowserPreviewPlaying]);
|
||||
|
||||
// OpenAI voice options
|
||||
const OPENAI_VOICE_OPTIONS = [
|
||||
{ value: 'alloy', label: 'Alloy' },
|
||||
{ value: 'ash', label: 'Ash' },
|
||||
{ value: 'ballad', label: 'Ballad' },
|
||||
{ value: 'coral', label: 'Coral' },
|
||||
{ value: 'echo', label: 'Echo' },
|
||||
{ value: 'fable', label: 'Fable' },
|
||||
{ value: 'nova', label: 'Nova' },
|
||||
{ value: 'onyx', label: 'Onyx' },
|
||||
{ value: 'sage', label: 'Sage' },
|
||||
{ value: 'shimmer', label: 'Shimmer' },
|
||||
{ value: 'verse', label: 'Verse' },
|
||||
{ value: 'marin', label: 'Marin' },
|
||||
{ value: 'cedar', label: 'Cedar' },
|
||||
];
|
||||
|
||||
// Check OpenAI TTS availability (including API key from settings)
|
||||
useEffect(() => {
|
||||
const checkOpenAIAvailability = async () => {
|
||||
try {
|
||||
// First check if server has API key configured
|
||||
const response = await fetch('/api/tts/status');
|
||||
const data = await response.json();
|
||||
console.log('[VoiceSettings] OpenAI TTS status:', data);
|
||||
|
||||
// Available if server has key OR user has set API key in settings
|
||||
const hasServerKey = data.available;
|
||||
const hasSettingsKey = openaiApiKey.trim().length > 0;
|
||||
setIsOpenAIAvailable(hasServerKey || hasSettingsKey);
|
||||
} catch (err) {
|
||||
console.error('[VoiceSettings] Failed to check OpenAI TTS status:', err);
|
||||
// Still available if user has set API key in settings
|
||||
setIsOpenAIAvailable(openaiApiKey.trim().length > 0);
|
||||
}
|
||||
};
|
||||
|
||||
checkOpenAIAvailability();
|
||||
}, [openaiApiKey]);
|
||||
|
||||
useEffect(() => {
|
||||
fetch('/api/tts/say/status')
|
||||
.then(res => res.json())
|
||||
.then(data => {
|
||||
console.log('[VoiceSettings] Say TTS status:', data);
|
||||
setIsSayAvailable(data.available);
|
||||
if (data.voices) {
|
||||
// Filter to unique voice names and sort alphabetically
|
||||
const uniqueVoices = data.voices
|
||||
.filter((v: { name: string; locale: string }, i: number, arr: Array<{ name: string; locale: string }>) =>
|
||||
arr.findIndex((x: { name: string }) => x.name === v.name) === i
|
||||
)
|
||||
.sort((a: { name: string }, b: { name: string }) => a.name.localeCompare(b.name));
|
||||
setSayVoices(uniqueVoices);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('[VoiceSettings] Failed to check Say TTS status:', err);
|
||||
setIsSayAvailable(false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Preview voice function
|
||||
const previewVoice = useCallback(async () => {
|
||||
// Stop any existing preview
|
||||
if (previewAudio) {
|
||||
previewAudio.pause();
|
||||
previewAudio.currentTime = 0;
|
||||
setPreviewAudio(null);
|
||||
setIsPreviewPlaying(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setIsPreviewPlaying(true);
|
||||
try {
|
||||
const response = await fetch('/api/tts/say/speak', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
text: `Hello! I'm ${sayVoice}. This is how I sound.`,
|
||||
voice: sayVoice,
|
||||
rate: Math.round(100 + (speechRate - 0.5) * 200),
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error('Preview failed');
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const audio = new Audio(url);
|
||||
|
||||
audio.onended = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
setPreviewAudio(null);
|
||||
setIsPreviewPlaying(false);
|
||||
};
|
||||
|
||||
audio.onerror = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
setPreviewAudio(null);
|
||||
setIsPreviewPlaying(false);
|
||||
};
|
||||
|
||||
setPreviewAudio(audio);
|
||||
await audio.play();
|
||||
} catch (err) {
|
||||
console.error('Voice preview failed:', err);
|
||||
setIsPreviewPlaying(false);
|
||||
}
|
||||
}, [sayVoice, speechRate, previewAudio]);
|
||||
|
||||
// Cleanup preview audio on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (previewAudio) {
|
||||
previewAudio.pause();
|
||||
}
|
||||
};
|
||||
}, [previewAudio]);
|
||||
|
||||
// Preview OpenAI voice
|
||||
const previewOpenAIVoice = useCallback(async () => {
|
||||
// Stop any existing preview
|
||||
if (openaiPreviewAudio) {
|
||||
openaiPreviewAudio.pause();
|
||||
openaiPreviewAudio.currentTime = 0;
|
||||
setOpenaiPreviewAudio(null);
|
||||
setIsOpenAIPreviewPlaying(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setIsOpenAIPreviewPlaying(true);
|
||||
try {
|
||||
const response = await fetch('/api/tts/speak', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
text: `Hello! I'm ${openaiVoice}. This is how I sound.`,
|
||||
voice: openaiVoice,
|
||||
speed: speechRate,
|
||||
apiKey: openaiApiKey || undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const audio = new Audio(url);
|
||||
|
||||
audio.onended = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
setOpenaiPreviewAudio(null);
|
||||
setIsOpenAIPreviewPlaying(false);
|
||||
};
|
||||
|
||||
audio.onerror = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
setOpenaiPreviewAudio(null);
|
||||
setIsOpenAIPreviewPlaying(false);
|
||||
};
|
||||
|
||||
setOpenaiPreviewAudio(audio);
|
||||
await audio.play();
|
||||
} catch (err) {
|
||||
console.error('[VoiceSettings] OpenAI voice preview failed:', err);
|
||||
setIsOpenAIPreviewPlaying(false);
|
||||
}
|
||||
}, [openaiVoice, speechRate, openaiPreviewAudio, openaiApiKey]);
|
||||
|
||||
// Cleanup OpenAI preview audio on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (openaiPreviewAudio) {
|
||||
openaiPreviewAudio.pause();
|
||||
}
|
||||
};
|
||||
}, [openaiPreviewAudio]);
|
||||
|
||||
return (
|
||||
<SettingsSection
|
||||
title="Voice"
|
||||
description="Configure voice conversation settings"
|
||||
>
|
||||
<div className="space-y-6">
|
||||
{/* Voice Mode Enable/Disable */}
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiMicLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Voice Mode
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Enable voice conversations with microphone input
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={voiceModeEnabled}
|
||||
onCheckedChange={setVoiceModeEnabled}
|
||||
aria-label="Toggle voice mode"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Voice provider selection - only show when voice mode is enabled */}
|
||||
{voiceModeEnabled && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiVolumeUpLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Voice Provider
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Choose your preferred text-to-speech provider
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2 flex-wrap justify-end">
|
||||
<Button
|
||||
variant={voiceProvider === 'browser' ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setVoiceProvider('browser')}
|
||||
className="min-w-[80px]"
|
||||
>
|
||||
Browser
|
||||
</Button>
|
||||
<Button
|
||||
variant={voiceProvider === 'openai' ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setVoiceProvider('openai')}
|
||||
className="min-w-[80px]"
|
||||
title={isOpenAIAvailable ? 'OpenAI voice' : 'OpenAI voice unavailable - API key not configured'}
|
||||
>
|
||||
OpenAI
|
||||
</Button>
|
||||
{isSayAvailable && (
|
||||
<Button
|
||||
variant={voiceProvider === 'say' ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setVoiceProvider('say')}
|
||||
className="min-w-[80px]"
|
||||
>
|
||||
<RiAppleLine className="w-4 h-4 mr-1" />
|
||||
Say
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Provider description */}
|
||||
{voiceModeEnabled && (
|
||||
<div className="p-3 rounded-lg bg-muted/50 border border-border/50">
|
||||
<p className="typography-micro text-muted-foreground">
|
||||
<span className="font-medium text-foreground">
|
||||
{voiceProvider === 'browser' ? 'Browser Voice:' : voiceProvider === 'openai' ? 'OpenAI:' : 'macOS Say:'}
|
||||
</span>{' '}
|
||||
{voiceProvider === 'browser'
|
||||
? 'Free, works offline, but has limited mobile support. Best for desktop use.'
|
||||
: voiceProvider === 'openai'
|
||||
? 'Higher quality voice synthesis that works reliably on mobile. Requires OpenAI API key.'
|
||||
: 'Native macOS speech synthesis. Free, fast, and works offline. Desktop only.'}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* OpenAI unavailable warning */}
|
||||
{voiceModeEnabled && voiceProvider === 'openai' && !isOpenAIAvailable && (
|
||||
<div className="flex items-start gap-3 p-3 rounded-lg bg-destructive/10 border border-destructive/20">
|
||||
<RiAlertLine className="w-5 h-5 text-destructive flex-shrink-0 mt-0.5" />
|
||||
<div className="space-y-1">
|
||||
<p className="typography-ui text-destructive font-medium">
|
||||
OpenAI voice unavailable
|
||||
</p>
|
||||
<p className="typography-micro text-destructive/80">
|
||||
OpenAI voice requires an OpenAI API key to be configured. Please set the OpenAI API key or switch to Browser voice.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* OpenAI API Key Input - show when OpenAI is selected or when no server key is configured */}
|
||||
{voiceModeEnabled && voiceProvider === 'openai' && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiKeyLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
OpenAI API Key
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
{isOpenAIAvailable && !openaiApiKey ? 'Using API key from OpenCode configuration' : 'Enter your OpenAI API key for voice synthesis'}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 w-[280px]">
|
||||
<input
|
||||
type="password"
|
||||
value={openaiApiKey}
|
||||
onChange={(e) => setOpenaiApiKey(e.target.value)}
|
||||
placeholder="sk-..."
|
||||
className="flex-1 px-3 py-2 text-sm bg-background border border-input rounded-md focus:outline-none focus:ring-2 focus:ring-ring"
|
||||
/>
|
||||
{openaiApiKey && (
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onClick={() => setOpenaiApiKey('')}
|
||||
title="Clear API key"
|
||||
>
|
||||
<RiCloseLine className="w-4 h-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* OpenAI Voice Selection */}
|
||||
{voiceModeEnabled && voiceProvider === 'openai' && isOpenAIAvailable && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiVolumeUpLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
OpenAI Voice
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Select an OpenAI voice for text-to-speech
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Select
|
||||
value={openaiVoice}
|
||||
onValueChange={setOpenaiVoice}
|
||||
>
|
||||
<SelectTrigger className="w-[160px]" aria-label="Select OpenAI voice">
|
||||
<SelectValue placeholder="Select voice" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="max-h-[300px]">
|
||||
{OPENAI_VOICE_OPTIONS.map((voice) => (
|
||||
<SelectItem key={voice.value} value={voice.value}>
|
||||
{voice.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="outline"
|
||||
onClick={previewOpenAIVoice}
|
||||
disabled={!isOpenAIAvailable}
|
||||
title={isOpenAIPreviewPlaying ? 'Stop preview' : 'Preview voice'}
|
||||
>
|
||||
{isOpenAIPreviewPlaying ? (
|
||||
<RiStopLine className="w-4 h-4" />
|
||||
) : (
|
||||
<RiPlayLine className="w-4 h-4" />
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* macOS Say Voice Selection */}
|
||||
{voiceModeEnabled && voiceProvider === 'say' && isSayAvailable && sayVoices.length > 0 && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiAppleLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
macOS Voice
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Select a voice installed on your Mac
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Select
|
||||
value={sayVoice}
|
||||
onValueChange={setSayVoice}
|
||||
>
|
||||
<SelectTrigger className="w-[160px]" aria-label="Select macOS voice">
|
||||
<SelectValue placeholder="Select voice" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="max-h-[300px]">
|
||||
{sayVoices.map((voice) => (
|
||||
<SelectItem key={voice.name} value={voice.name}>
|
||||
{voice.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="outline"
|
||||
onClick={previewVoice}
|
||||
title={isPreviewPlaying ? 'Stop preview' : 'Preview voice'}
|
||||
>
|
||||
{isPreviewPlaying ? (
|
||||
<RiStopLine className="w-4 h-4" />
|
||||
) : (
|
||||
<RiPlayLine className="w-4 h-4" />
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Browser Voice Selection */}
|
||||
{voiceModeEnabled && voiceProvider === 'browser' && filteredBrowserVoices.length > 0 && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiChromeLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Browser Voice
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Select a voice from your browser
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Select
|
||||
value={browserVoice || '__auto__'}
|
||||
onValueChange={(value) => setBrowserVoice(value === '__auto__' ? '' : value)}
|
||||
>
|
||||
<SelectTrigger className="w-[200px]" aria-label="Select browser voice">
|
||||
<SelectValue placeholder="Auto (default)" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="max-h-[300px]">
|
||||
<SelectItem value="__auto__">Auto (default)</SelectItem>
|
||||
{filteredBrowserVoices.map((voice) => (
|
||||
<SelectItem key={voice.name} value={voice.name}>
|
||||
{voice.name} ({voice.lang})
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="outline"
|
||||
onClick={previewBrowserVoice}
|
||||
title={isBrowserPreviewPlaying ? 'Stop preview' : 'Preview voice'}
|
||||
>
|
||||
{isBrowserPreviewPlaying ? (
|
||||
<RiStopLine className="w-4 h-4" />
|
||||
) : (
|
||||
<RiPlayLine className="w-4 h-4" />
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Language selection */}
|
||||
{voiceModeEnabled && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiMicLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Language
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Language for speech recognition and synthesis
|
||||
</p>
|
||||
</div>
|
||||
<Select
|
||||
value={language}
|
||||
onValueChange={setLanguage}
|
||||
disabled={!isSupported}
|
||||
>
|
||||
<SelectTrigger className="w-[180px]" aria-label="Select language">
|
||||
<SelectValue placeholder="Select language" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{LANGUAGE_OPTIONS.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value}>
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Show TTS buttons on messages */}
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiVolumeUpLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Message Read Aloud
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Show speaker button on AI responses to read them aloud
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={showMessageTTSButtons}
|
||||
onCheckedChange={setShowMessageTTSButtons}
|
||||
aria-label="Toggle message TTS buttons"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Summarization Section */}
|
||||
<div className="pt-4 border-t border-border/40 space-y-4">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<RiFileTextLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground font-medium">
|
||||
Summarization
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Summarize Message TTS */}
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Summarize Message Playback
|
||||
</span>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Summarize long messages before reading them aloud
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={summarizeMessageTTS}
|
||||
onCheckedChange={setSummarizeMessageTTS}
|
||||
aria-label="Toggle message TTS summarization"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Summarize Voice Conversation */}
|
||||
{voiceModeEnabled && (
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Summarize Voice Responses
|
||||
</span>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Summarize long AI responses during voice conversations
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={summarizeVoiceConversation}
|
||||
onCheckedChange={setSummarizeVoiceConversation}
|
||||
aria-label="Toggle voice conversation summarization"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Character Threshold - only show if either summarization is enabled */}
|
||||
{(summarizeMessageTTS || summarizeVoiceConversation) && (
|
||||
<>
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Character Threshold
|
||||
</span>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Summarize text longer than this ({summarizeCharacterThreshold} chars)
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-[180px]">
|
||||
<Slider
|
||||
value={summarizeCharacterThreshold}
|
||||
onChange={setSummarizeCharacterThreshold}
|
||||
min={50}
|
||||
max={2000}
|
||||
step={50}
|
||||
label="Character threshold"
|
||||
valueFormatter={(v: number) => `${v}`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Summary Length Limit
|
||||
</span>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Max characters for the summary ({summarizeMaxLength} chars)
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-[180px]">
|
||||
<Slider
|
||||
value={summarizeMaxLength}
|
||||
onChange={setSummarizeMaxLength}
|
||||
min={50}
|
||||
max={2000}
|
||||
step={50}
|
||||
label="Summary length limit"
|
||||
valueFormatter={(v: number) => `${v}`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Speech Rate */}
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiSpeedLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Speech Rate
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Speed of speech (0.5x - 2x)
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-[180px]">
|
||||
<Slider
|
||||
value={speechRate}
|
||||
onChange={setSpeechRate}
|
||||
min={0.5}
|
||||
max={2}
|
||||
step={0.1}
|
||||
disabled={!isSupported}
|
||||
label="Speech rate"
|
||||
valueFormatter={(v: number) => `${v.toFixed(1)}x`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Speech Pitch */}
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiMusicLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Speech Pitch
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Voice pitch (0.5 - 2)
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-[180px]">
|
||||
<Slider
|
||||
value={speechPitch}
|
||||
onChange={setSpeechPitch}
|
||||
min={0.5}
|
||||
max={2}
|
||||
step={0.1}
|
||||
disabled={!isSupported}
|
||||
label="Speech pitch"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Speech Volume */}
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<RiSoundModuleLine className="w-4 h-4 text-muted-foreground" />
|
||||
<span className="typography-ui-label text-foreground">
|
||||
Speech Volume
|
||||
</span>
|
||||
</div>
|
||||
<p className="typography-meta text-muted-foreground">
|
||||
Voice volume (0 - 100%)
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-[180px]">
|
||||
<Slider
|
||||
value={speechVolume}
|
||||
onChange={setSpeechVolume}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.1}
|
||||
disabled={!isSupported}
|
||||
label="Speech volume"
|
||||
valueFormatter={(v: number) => `${Math.round(v * 100)}%`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Keyboard shortcut hint */}
|
||||
{voiceModeEnabled && isSupported && (
|
||||
<div className="pt-4 border-t border-border/40">
|
||||
<p className="typography-micro text-muted-foreground">
|
||||
<span className="font-medium text-foreground">Tip:</span>{' '}
|
||||
Press <kbd className="px-1.5 py-0.5 rounded bg-muted typography-mono text-xs">Shift</kbd> +{' '}
|
||||
<kbd className="px-1.5 py-0.5 rounded bg-muted typography-mono text-xs">Click</kbd>{' '}
|
||||
on the voice button to quickly toggle continuous mode
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</SettingsSection>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,381 @@
|
||||
/**
|
||||
* BrowserVoiceButton Component
|
||||
*
|
||||
* Voice toggle button for browser-based voice chat with language selection.
|
||||
* Shows visual state indicators for different voice modes.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <BrowserVoiceButton />
|
||||
* ```
|
||||
*/
|
||||
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { useBrowserVoice } from '@/hooks/useBrowserVoice';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { isVSCodeRuntime } from '@/lib/desktop';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipProvider,
|
||||
TooltipTrigger,
|
||||
} from '@/components/ui/tooltip';
|
||||
import {
|
||||
RiMicOffLine,
|
||||
RiStopCircleLine,
|
||||
RiVoiceRecognitionLine,
|
||||
RiVolumeUpLine,
|
||||
} from '@remixicon/react';
|
||||
import { VoiceStatusIndicator } from './VoiceStatusIndicator';
|
||||
import { toast } from '@/components/ui/toast';
|
||||
|
||||
// Status text for accessibility and labels
|
||||
const statusLabels: Record<string, string> = {
|
||||
idle: 'Start Voice',
|
||||
listening: 'Listening',
|
||||
processing: 'Processing',
|
||||
speaking: 'AI Speaking',
|
||||
error: 'Voice Error',
|
||||
};
|
||||
|
||||
// iOS Safari detection utility
|
||||
const isIOSSafari = (): boolean => {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
const isIOS = /iphone|ipad|ipod/i.test(userAgent);
|
||||
const isSafari = /safari/i.test(userAgent) && !/chrome|crios|crmo/i.test(userAgent);
|
||||
return isIOS && isSafari;
|
||||
};
|
||||
|
||||
const normalizeVoiceErrorMessage = (error: string): string => {
|
||||
const isMediaDevicesError =
|
||||
error.includes('getUserMedia') ||
|
||||
error.includes('mediaDevices') ||
|
||||
error.includes('Cannot read properties of undefined');
|
||||
|
||||
if (!isMediaDevicesError) {
|
||||
return error;
|
||||
}
|
||||
|
||||
if (typeof window !== 'undefined' && !window.isSecureContext) {
|
||||
return 'Voice requires a secure connection (HTTPS) or localhost. Please use HTTPS or access via localhost.';
|
||||
}
|
||||
|
||||
return 'Microphone access is unavailable in this runtime. On desktop, check System Settings -> Privacy & Security -> Microphone for OpenChamber.';
|
||||
};
|
||||
|
||||
/**
|
||||
* Browser Voice Button with language selection
|
||||
*/
|
||||
export function BrowserVoiceButton() {
|
||||
const voiceModeEnabled = useConfigStore((s) => s.voiceModeEnabled);
|
||||
|
||||
const {
|
||||
status,
|
||||
isSupported,
|
||||
error,
|
||||
|
||||
startVoice,
|
||||
stopVoice,
|
||||
conversationMode,
|
||||
toggleConversationMode,
|
||||
isMobile,
|
||||
} = useBrowserVoice();
|
||||
|
||||
const [isPressing, setIsPressing] = useState(false);
|
||||
const isVSCode = isVSCodeRuntime();
|
||||
const buttonSizeClass = isMobile ? 'h-8 w-8 min-h-[32px] min-w-[32px]' : (isVSCode ? 'h-5 w-5' : 'h-6 w-6');
|
||||
const iconSizeClass = isMobile ? 'h-[18px] w-[18px]' : (isVSCode ? 'h-4 w-4' : 'h-[18px] w-[18px]');
|
||||
const continuousIconSizeClass = 'size-[18px]';
|
||||
const clearHoverBackgroundClass = 'bg-transparent hover:!bg-transparent active:!bg-transparent focus-visible:!bg-transparent';
|
||||
|
||||
// Refs for touch handling
|
||||
const touchHandledRef = useRef(false);
|
||||
const isIOSSafariRef = useRef(false);
|
||||
const longPressTimerRef = useRef<NodeJS.Timeout | null>(null);
|
||||
const longPressTriggeredRef = useRef(false);
|
||||
const lastToastedErrorRef = useRef<string | null>(null);
|
||||
|
||||
// Initialize iOS detection on mount
|
||||
useEffect(() => {
|
||||
isIOSSafariRef.current = isIOSSafari();
|
||||
}, []);
|
||||
|
||||
// NOTE: Do NOT pre-request microphone permission on mount.
|
||||
// Permission is requested when the user explicitly taps the mic button.
|
||||
// Pre-requesting causes an unwanted permission prompt on mobile page load.
|
||||
|
||||
// Determine active states
|
||||
const isActive = status === 'listening' || status === 'speaking' || status === 'processing';
|
||||
const isError = status === 'error';
|
||||
const isIdle = status === 'idle';
|
||||
|
||||
const isSpeaking = status === 'speaking';
|
||||
|
||||
// Show toast notification when voice error occurs
|
||||
useEffect(() => {
|
||||
if (isError && error) {
|
||||
if (lastToastedErrorRef.current === error) {
|
||||
return;
|
||||
}
|
||||
lastToastedErrorRef.current = error;
|
||||
const displayError = normalizeVoiceErrorMessage(error);
|
||||
|
||||
toast.error(displayError, {
|
||||
duration: 5000,
|
||||
});
|
||||
}
|
||||
|
||||
if (!isError) {
|
||||
lastToastedErrorRef.current = null;
|
||||
}
|
||||
}, [isError, error]);
|
||||
|
||||
// Status text for accessibility
|
||||
const statusText = isError
|
||||
? error || 'Voice Error'
|
||||
: conversationMode && status === 'idle'
|
||||
? 'Start Voice (Continuous mode on)'
|
||||
: statusLabels[status] || 'Start Voice';
|
||||
|
||||
// Tooltip content based on state
|
||||
const getTooltipContent = () => {
|
||||
if (isError && error) {
|
||||
return normalizeVoiceErrorMessage(error);
|
||||
}
|
||||
if (isActive) {
|
||||
return 'Stop voice conversation';
|
||||
}
|
||||
if (isMobile) {
|
||||
return 'Start voice conversation';
|
||||
}
|
||||
return `Start voice conversation (Shift+Click for continuous mode) • Cmd/Ctrl+Shift+V to toggle`;
|
||||
};
|
||||
|
||||
// Handle voice activation (used by both click and touch)
|
||||
const activateVoice = useCallback(async () => {
|
||||
if (isActive) {
|
||||
stopVoice();
|
||||
} else if (status !== 'error') {
|
||||
// On mobile, we must NOT do any async operations before calling startVoice()
|
||||
// because iOS Safari requires SpeechRecognition.start() to be called
|
||||
// synchronously within the user gesture handler
|
||||
if (isMobile) {
|
||||
// Start voice immediately - no await before this!
|
||||
// Audio unlock is now handled inside startVoice() for mobile
|
||||
startVoice();
|
||||
} else {
|
||||
// Desktop can use async path
|
||||
try {
|
||||
await startVoice();
|
||||
} catch (err) {
|
||||
console.error('Failed to start voice:', err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Reset from error state
|
||||
if (isMobile) {
|
||||
startVoice();
|
||||
} else {
|
||||
try {
|
||||
await startVoice();
|
||||
} catch (err) {
|
||||
console.error('Failed to start voice:', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [isActive, status, startVoice, stopVoice, isMobile]);
|
||||
|
||||
// Handle Shift+Click to toggle conversation mode
|
||||
const handleClick = useCallback(async (e: React.MouseEvent) => {
|
||||
// Prevent double-firing if touch already handled this
|
||||
if (touchHandledRef.current) {
|
||||
touchHandledRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// Shift+Click toggles conversation mode
|
||||
if (e.shiftKey) {
|
||||
toggleConversationMode();
|
||||
return;
|
||||
}
|
||||
|
||||
await activateVoice();
|
||||
}, [activateVoice, toggleConversationMode]);
|
||||
|
||||
// Handle touch start for mobile devices
|
||||
const handleTouchStart = useCallback((e: React.TouchEvent) => {
|
||||
// Prevent default to stop mouse event emulation
|
||||
e.preventDefault();
|
||||
|
||||
// Mark that touch handled this interaction
|
||||
touchHandledRef.current = true;
|
||||
longPressTriggeredRef.current = false;
|
||||
|
||||
// Immediate visual feedback
|
||||
setIsPressing(true);
|
||||
|
||||
// Set up long-press timer for toggling conversation mode (500ms)
|
||||
longPressTimerRef.current = setTimeout(() => {
|
||||
longPressTriggeredRef.current = true;
|
||||
toggleConversationMode();
|
||||
// Haptic feedback if available
|
||||
if (navigator.vibrate) {
|
||||
navigator.vibrate(50);
|
||||
}
|
||||
setIsPressing(false);
|
||||
}, 500);
|
||||
}, [toggleConversationMode]);
|
||||
|
||||
// Handle touch end
|
||||
const handleTouchEnd = useCallback(() => {
|
||||
// Clear long-press timer
|
||||
if (longPressTimerRef.current) {
|
||||
clearTimeout(longPressTimerRef.current);
|
||||
longPressTimerRef.current = null;
|
||||
}
|
||||
|
||||
// Only activate voice if long-press wasn't triggered
|
||||
if (!longPressTriggeredRef.current) {
|
||||
activateVoice();
|
||||
}
|
||||
|
||||
setIsPressing(false);
|
||||
}, [activateVoice]);
|
||||
|
||||
// Handle touch cancel
|
||||
const handleTouchCancel = useCallback(() => {
|
||||
if (longPressTimerRef.current) {
|
||||
clearTimeout(longPressTimerRef.current);
|
||||
longPressTimerRef.current = null;
|
||||
}
|
||||
setIsPressing(false);
|
||||
}, []);
|
||||
|
||||
const handleToggleConversationMode = useCallback((event: React.MouseEvent<HTMLButtonElement>) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
toggleConversationMode();
|
||||
}, [toggleConversationMode]);
|
||||
|
||||
|
||||
|
||||
// If voice mode is disabled, don't render anything
|
||||
if (!voiceModeEnabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// If not supported, show disabled button with tooltip
|
||||
if (!isSupported) {
|
||||
const supportDetails = browserVoiceService.getSupportDetails();
|
||||
const tooltipMessage = !supportDetails.secureContext
|
||||
? 'Voice requires HTTPS or localhost. Please use a secure connection.'
|
||||
: !supportDetails.recognition
|
||||
? 'Speech recognition not supported in this browser. Try Chrome, Edge, or Safari.'
|
||||
: !supportDetails.synthesis
|
||||
? 'Speech synthesis not supported in this browser.'
|
||||
: 'Voice not supported in this browser';
|
||||
|
||||
return (
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
disabled
|
||||
aria-label={tooltipMessage}
|
||||
className={`${buttonSizeClass} p-0 ${clearHoverBackgroundClass}`}
|
||||
>
|
||||
<RiMicOffLine className={`${iconSizeClass} opacity-50`} />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" align="center">
|
||||
<p className="max-w-[200px] text-center">{tooltipMessage}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={`flex items-center ${isMobile ? 'gap-1' : 'gap-1.5'}`}>
|
||||
{/* Status indicator with label - show when active, simplified on mobile */}
|
||||
{isActive && !isMobile && (
|
||||
<VoiceStatusIndicator
|
||||
status={status}
|
||||
showLabel
|
||||
size="sm"
|
||||
className="mr-1"
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Voice button with tooltip */}
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onClick={handleClick}
|
||||
onTouchStart={handleTouchStart}
|
||||
onTouchEnd={handleTouchEnd}
|
||||
onTouchCancel={handleTouchCancel}
|
||||
aria-label={statusText}
|
||||
className={`
|
||||
relative
|
||||
${buttonSizeClass}
|
||||
p-0
|
||||
${clearHoverBackgroundClass}
|
||||
touch-manipulation
|
||||
${isPressing ? 'scale-95 opacity-80' : ''}
|
||||
${conversationMode && isIdle && isMobile ? 'ring-1 ring-primary/50' : ''}
|
||||
`}
|
||||
style={{
|
||||
WebkitTapHighlightColor: 'transparent',
|
||||
touchAction: 'manipulation',
|
||||
}}
|
||||
>
|
||||
{isActive ? (
|
||||
isSpeaking ? (
|
||||
// Green speaker icon when AI is speaking
|
||||
<RiVolumeUpLine className={`${iconSizeClass} text-green-400 animate-pulse`} />
|
||||
) : (
|
||||
// Red stop icon for listening/processing (both mobile and desktop)
|
||||
<RiStopCircleLine className={`${iconSizeClass} text-[var(--status-error)]`} />
|
||||
)
|
||||
) : (
|
||||
<VoiceStatusIndicator
|
||||
status={isError ? 'idle' : status}
|
||||
size={isMobile || isVSCode ? 'sm' : 'md'}
|
||||
/>
|
||||
)}
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" align="center">
|
||||
<p className="max-w-[200px] text-center">{getTooltipContent()}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
|
||||
{/* Conversation mode toggle button */}
|
||||
{(status === 'idle' || status === 'error') && (
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onPointerDownCapture={(event) => event.stopPropagation()}
|
||||
onClick={handleToggleConversationMode}
|
||||
aria-label={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
|
||||
title={conversationMode ? 'Continuous mode on' : 'Continuous mode off'}
|
||||
className={
|
||||
`${buttonSizeClass} p-0 ${clearHoverBackgroundClass} ${conversationMode ? 'text-[var(--status-info)] hover:text-[var(--status-info)]' : 'text-muted-foreground hover:text-foreground'}`
|
||||
}
|
||||
>
|
||||
<RiVoiceRecognitionLine className={continuousIconSizeClass} />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import React from 'react';
|
||||
import { useVoiceContext } from '@/hooks/useVoiceContext';
|
||||
|
||||
/**
|
||||
* Provider component that initializes voice context sync.
|
||||
* Wrap the app with this to enable voice session awareness.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <VoiceProvider>
|
||||
* <App />
|
||||
* </VoiceProvider>
|
||||
* ```
|
||||
*/
|
||||
export function VoiceProvider({ children }: { children: React.ReactNode }) {
|
||||
// Activate session-to-voice sync
|
||||
useVoiceContext();
|
||||
|
||||
return <>{children}</>;
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* VoiceStatusIndicator Component
|
||||
*
|
||||
* Reusable visual indicator for voice mode states with icons, animations,
|
||||
* and optional status text labels.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* // Basic usage - icon only
|
||||
* <VoiceStatusIndicator status="listening" />
|
||||
*
|
||||
* // With label
|
||||
* <VoiceStatusIndicator status="listening" showLabel />
|
||||
*
|
||||
* // Different size
|
||||
* <VoiceStatusIndicator status="processing" size="lg" />
|
||||
* ```
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import {
|
||||
RiMicLine,
|
||||
RiMicOffLine,
|
||||
RiLoader4Line,
|
||||
RiVolumeUpLine,
|
||||
RiAlertLine,
|
||||
} from '@remixicon/react';
|
||||
import type { BrowserVoiceStatus } from '@/hooks/useBrowserVoice';
|
||||
|
||||
export interface VoiceStatusIndicatorProps {
|
||||
/** Current voice status */
|
||||
status: BrowserVoiceStatus;
|
||||
/** Show text label next to icon */
|
||||
showLabel?: boolean;
|
||||
/** Size of the indicator */
|
||||
size?: 'sm' | 'md' | 'lg';
|
||||
/** Optional className for styling */
|
||||
className?: string;
|
||||
/** Whether conversation mode is active (shows indicator dot when idle) */
|
||||
conversationMode?: boolean;
|
||||
}
|
||||
|
||||
const sizeClasses = {
|
||||
sm: {
|
||||
icon: 'w-4 h-4',
|
||||
container: 'gap-1.5',
|
||||
},
|
||||
md: {
|
||||
icon: 'w-5 h-5',
|
||||
container: 'gap-2',
|
||||
},
|
||||
lg: {
|
||||
icon: 'w-6 h-6',
|
||||
container: 'gap-2.5',
|
||||
},
|
||||
};
|
||||
|
||||
const statusConfig: Record<
|
||||
BrowserVoiceStatus,
|
||||
{
|
||||
icon: typeof RiMicLine;
|
||||
color: string;
|
||||
label: string;
|
||||
animation?: string;
|
||||
}
|
||||
> = {
|
||||
idle: {
|
||||
icon: RiMicOffLine,
|
||||
color: 'text-muted-foreground',
|
||||
label: 'Voice Ready',
|
||||
},
|
||||
listening: {
|
||||
icon: RiMicLine,
|
||||
color: 'text-primary',
|
||||
label: 'Listening...',
|
||||
animation: 'animate-pulse',
|
||||
},
|
||||
processing: {
|
||||
icon: RiLoader4Line,
|
||||
color: 'text-primary',
|
||||
label: 'Processing...',
|
||||
animation: 'animate-spin',
|
||||
},
|
||||
speaking: {
|
||||
icon: RiVolumeUpLine,
|
||||
color: 'text-green-500',
|
||||
label: 'Speaking...',
|
||||
},
|
||||
error: {
|
||||
icon: RiAlertLine,
|
||||
color: 'text-destructive',
|
||||
label: 'Voice Error',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* VoiceStatusIndicator - Visual indicator for voice mode states
|
||||
*/
|
||||
export function VoiceStatusIndicator({
|
||||
status,
|
||||
showLabel = false,
|
||||
size = 'md',
|
||||
className = '',
|
||||
conversationMode = false,
|
||||
}: VoiceStatusIndicatorProps) {
|
||||
const config = statusConfig[status];
|
||||
const Icon = config.icon;
|
||||
const sizeClass = sizeClasses[size];
|
||||
const containerClass = showLabel ? sizeClass.container : '';
|
||||
|
||||
return (
|
||||
<div className={`flex items-center ${containerClass} ${className}`}>
|
||||
<div className="relative">
|
||||
<Icon
|
||||
className={`
|
||||
${sizeClass.icon}
|
||||
${config.color}
|
||||
${config.animation || ''}
|
||||
`}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
{/* Conversation mode indicator dot - only when idle and conversation mode is on */}
|
||||
{conversationMode && status === 'idle' && (
|
||||
<span
|
||||
className="absolute -top-0.5 -right-0.5 w-2 h-2 bg-green-500 rounded-full"
|
||||
aria-label="Conversation mode active"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
{showLabel && (
|
||||
<span className={`typography-meta ${config.color}`}>
|
||||
{config.label}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default VoiceStatusIndicator;
|
||||
@@ -0,0 +1,3 @@
|
||||
export { VoiceProvider } from './VoiceProvider';
|
||||
export { BrowserVoiceButton } from './BrowserVoiceButton';
|
||||
export { VoiceStatusIndicator } from './VoiceStatusIndicator';
|
||||
@@ -0,0 +1,792 @@
|
||||
/**
|
||||
* useBrowserVoice Hook
|
||||
*
|
||||
* React hook for browser-based voice chat integration.
|
||||
* Manages speech recognition, AI message sending, and speech synthesis.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const {
|
||||
* status,
|
||||
* isSupported,
|
||||
* language,
|
||||
* setLanguage,
|
||||
* startVoice,
|
||||
* stopVoice,
|
||||
* prepareVoice,
|
||||
* isMobile,
|
||||
* } = useBrowserVoice();
|
||||
*
|
||||
* // Start voice mode
|
||||
* startVoice();
|
||||
*
|
||||
* // Change language
|
||||
* setLanguage('es-ES');
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { useSessionStore } from '@/stores/useSessionStore';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useServerTTS } from './useServerTTS';
|
||||
import { useSayTTS } from './useSayTTS';
|
||||
import { summarizeText, shouldSummarize, sanitizeForTTS } from '@/lib/voice/summarize';
|
||||
|
||||
export type BrowserVoiceStatus = 'idle' | 'listening' | 'processing' | 'speaking' | 'error';
|
||||
|
||||
export interface UseBrowserVoiceReturn {
|
||||
/** Current voice status */
|
||||
status: BrowserVoiceStatus;
|
||||
/** Whether browser voice is supported */
|
||||
isSupported: boolean;
|
||||
/** Error message if any */
|
||||
error: string | null;
|
||||
/** Current language for recognition/synthesis */
|
||||
language: string;
|
||||
/** Set language for voice operations */
|
||||
setLanguage: (lang: string) => void;
|
||||
/** Start voice mode (listening) */
|
||||
startVoice: () => void;
|
||||
/** Stop voice mode */
|
||||
stopVoice: () => void;
|
||||
/** Whether conversation mode is active */
|
||||
conversationMode: boolean;
|
||||
/** Toggle conversation mode */
|
||||
toggleConversationMode: () => void;
|
||||
/** Prepare voice for mobile (request permission) */
|
||||
prepareVoice: () => Promise<boolean>;
|
||||
/** Whether the device is mobile */
|
||||
isMobile: boolean;
|
||||
/** Current voice provider */
|
||||
voiceProvider: 'browser' | 'openai' | 'say';
|
||||
}
|
||||
|
||||
// Storage key for persisting language preference
|
||||
const LANGUAGE_STORAGE_KEY = 'browserVoiceLanguage';
|
||||
// Storage key for persisting conversation mode preference
|
||||
const CONVERSATION_MODE_STORAGE_KEY = 'browserVoiceConversationMode';
|
||||
const LANGUAGE_CHANGE_EVENT = 'openchamber:voice-language-changed';
|
||||
const CONVERSATION_MODE_CHANGE_EVENT = 'openchamber:voice-conversation-mode-changed';
|
||||
const FINAL_TRANSCRIPT_SETTLE_MS = 1200;
|
||||
const DEVICE_CHANGE_RESTART_DELAY_MS = 700;
|
||||
const BLOCKED_SPEECH_LANGUAGES = new Set(['ru', 'ru-RU']);
|
||||
|
||||
const sanitizeSpeechLanguage = (lang: string): string => {
|
||||
const normalized = (lang || '').trim();
|
||||
if (!normalized) {
|
||||
return 'en-US';
|
||||
}
|
||||
const base = normalized.split('-')[0].toLowerCase();
|
||||
if (BLOCKED_SPEECH_LANGUAGES.has(normalized) || BLOCKED_SPEECH_LANGUAGES.has(base)) {
|
||||
return 'en-US';
|
||||
}
|
||||
return normalized;
|
||||
};
|
||||
|
||||
/**
|
||||
* Hook for managing browser-based voice conversations
|
||||
*/
|
||||
export function useBrowserVoice(): UseBrowserVoiceReturn {
|
||||
const [status, setStatus] = useState<BrowserVoiceStatus>('idle');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [language, setLanguageState] = useState<string>(() => {
|
||||
// Try to load from localStorage, fallback to navigator.language
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem(LANGUAGE_STORAGE_KEY);
|
||||
if (saved) return sanitizeSpeechLanguage(saved);
|
||||
}
|
||||
return sanitizeSpeechLanguage(navigator.language || 'en-US');
|
||||
});
|
||||
const [conversationMode, setConversationModeState] = useState<boolean>(() => {
|
||||
// Try to load from localStorage, default to false
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem(CONVERSATION_MODE_STORAGE_KEY);
|
||||
return saved === 'true';
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
const isSupported = browserVoiceService.isSupported();
|
||||
|
||||
// Mobile detection
|
||||
const isMobile = useMemo(() => {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
return /iphone|ipad|ipod|android|mobile|webos|blackberry|iemobile|opera mini/i.test(userAgent);
|
||||
}, []);
|
||||
|
||||
// Refs for managing async operations
|
||||
const isActiveRef = useRef(false);
|
||||
const processingMessageRef = useRef(false);
|
||||
const lastTranscriptRef = useRef('');
|
||||
const messagesRef = useRef<Map<string, { info: { role: string }; parts: Array<{ type: string; text?: string }> }>>(new Map());
|
||||
const pendingResumeOnVisibleRef = useRef(false);
|
||||
const pendingFinalTranscriptRef = useRef('');
|
||||
const finalTranscriptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const deviceChangeRestartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Store access
|
||||
const currentSessionId = useSessionStore((s) => s.currentSessionId);
|
||||
const sendMessage = useSessionStore((s) => s.sendMessage);
|
||||
const setPendingInputText = useSessionStore((s) => s.setPendingInputText);
|
||||
const messages = useSessionStore((s) => s.messages);
|
||||
const createSession = useSessionStore((s) => s.createSession);
|
||||
const { currentProviderId, currentModelId, currentAgentName, voiceProvider, speechRate, speechPitch, speechVolume, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold } = useConfigStore();
|
||||
|
||||
// Server TTS for mobile (bypasses Safari audio restrictions)
|
||||
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable, unlockAudio: unlockServerTTSAudio } = useServerTTS();
|
||||
|
||||
// macOS Say TTS
|
||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable, unlockAudio: unlockSayTTSAudio } = useSayTTS();
|
||||
|
||||
// Update messages ref when messages change
|
||||
useEffect(() => {
|
||||
if (currentSessionId) {
|
||||
const sessionMessages = messages.get(currentSessionId);
|
||||
if (sessionMessages) {
|
||||
messagesRef.current = new Map(sessionMessages.map(m => [m.info.id, m]));
|
||||
}
|
||||
}
|
||||
}, [messages, currentSessionId]);
|
||||
|
||||
// Stop voice when session changes to prevent microphone from staying active
|
||||
// This ensures voice mode doesn't carry over between sessions
|
||||
const prevSessionIdRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (prevSessionIdRef.current !== null && prevSessionIdRef.current !== currentSessionId) {
|
||||
// Session changed - stop any active voice session
|
||||
if (isActiveRef.current) {
|
||||
console.log('[useBrowserVoice] Session changed, stopping voice');
|
||||
isActiveRef.current = false;
|
||||
processingMessageRef.current = false;
|
||||
browserVoiceService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
setStatus('idle');
|
||||
setError(null);
|
||||
}
|
||||
}
|
||||
prevSessionIdRef.current = currentSessionId;
|
||||
}, [currentSessionId]);
|
||||
|
||||
// Persist language preference
|
||||
const setLanguage = useCallback((lang: string) => {
|
||||
const nextLang = sanitizeSpeechLanguage(lang);
|
||||
setLanguageState(nextLang);
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem(LANGUAGE_STORAGE_KEY, nextLang);
|
||||
window.dispatchEvent(new CustomEvent<string>(LANGUAGE_CHANGE_EVENT, { detail: nextLang }));
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof window === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleLanguageEvent = (event: Event) => {
|
||||
const customEvent = event as CustomEvent<string>;
|
||||
const nextLang = sanitizeSpeechLanguage(customEvent.detail || localStorage.getItem(LANGUAGE_STORAGE_KEY) || 'en-US');
|
||||
setLanguageState((prev) => (prev === nextLang ? prev : nextLang));
|
||||
};
|
||||
|
||||
const handleStorage = (event: StorageEvent) => {
|
||||
if (event.key !== LANGUAGE_STORAGE_KEY || !event.newValue) {
|
||||
return;
|
||||
}
|
||||
const nextLang = sanitizeSpeechLanguage(event.newValue);
|
||||
setLanguageState((prev) => (prev === nextLang ? prev : nextLang));
|
||||
};
|
||||
|
||||
window.addEventListener(LANGUAGE_CHANGE_EVENT, handleLanguageEvent as EventListener);
|
||||
window.addEventListener('storage', handleStorage);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener(LANGUAGE_CHANGE_EVENT, handleLanguageEvent as EventListener);
|
||||
window.removeEventListener('storage', handleStorage);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Toggle conversation mode
|
||||
const toggleConversationMode = useCallback(() => {
|
||||
setConversationModeState((prev) => {
|
||||
const next = !prev;
|
||||
browserVoiceService.setConversationMode(next);
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem(CONVERSATION_MODE_STORAGE_KEY, String(next));
|
||||
window.dispatchEvent(new CustomEvent<boolean>(CONVERSATION_MODE_CHANGE_EVENT, { detail: next }));
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof window === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleConversationModeEvent = (event: Event) => {
|
||||
const customEvent = event as CustomEvent<boolean>;
|
||||
const detail = customEvent.detail;
|
||||
if (typeof detail !== 'boolean') {
|
||||
return;
|
||||
}
|
||||
setConversationModeState((prev) => (prev === detail ? prev : detail));
|
||||
};
|
||||
|
||||
const handleStorage = (event: StorageEvent) => {
|
||||
if (event.key !== CONVERSATION_MODE_STORAGE_KEY || event.newValue == null) {
|
||||
return;
|
||||
}
|
||||
const next = event.newValue === 'true';
|
||||
setConversationModeState((prev) => (prev === next ? prev : next));
|
||||
};
|
||||
|
||||
window.addEventListener(CONVERSATION_MODE_CHANGE_EVENT, handleConversationModeEvent as EventListener);
|
||||
window.addEventListener('storage', handleStorage);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener(CONVERSATION_MODE_CHANGE_EVENT, handleConversationModeEvent as EventListener);
|
||||
window.removeEventListener('storage', handleStorage);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Initialize conversation mode in service on mount
|
||||
useEffect(() => {
|
||||
browserVoiceService.setConversationMode(conversationMode);
|
||||
}, [conversationMode]);
|
||||
|
||||
// Refs for callbacks to avoid circular dependencies
|
||||
const handleSpeechErrorRef = useRef<((errorMsg: string) => void) | null>(null);
|
||||
const handleSpeechResultRef = useRef<((text: string, isFinal: boolean) => Promise<void>) | null>(null);
|
||||
|
||||
// Handle speech recognition error
|
||||
const handleSpeechError = useCallback((errorMsg: string) => {
|
||||
// Ignore errors if we've already stopped voice mode
|
||||
if (!isActiveRef.current) {
|
||||
console.log('[useBrowserVoice] Ignoring error after voice stopped:', errorMsg);
|
||||
return;
|
||||
}
|
||||
|
||||
const normalizedError = errorMsg.toLowerCase();
|
||||
if (normalizedError.includes('aborted')) {
|
||||
console.log('[useBrowserVoice] Ignoring non-fatal aborted error');
|
||||
setError(null);
|
||||
setStatus('listening');
|
||||
return;
|
||||
}
|
||||
|
||||
const isHidden = typeof document !== 'undefined' && document.visibilityState !== 'visible';
|
||||
const isPermissionStyleError =
|
||||
normalizedError.includes('permission') ||
|
||||
normalizedError.includes('not allowed') ||
|
||||
normalizedError.includes('service not allowed');
|
||||
|
||||
if (isHidden && isPermissionStyleError && conversationMode) {
|
||||
console.log('[useBrowserVoice] Suppressing permission error while app hidden; will resume on visibility');
|
||||
pendingResumeOnVisibleRef.current = true;
|
||||
setError(null);
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
console.error('[useBrowserVoice] Recognition error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
|
||||
// Auto-recover from certain errors
|
||||
if (!errorMsg.includes('permission') && !errorMsg.includes('not allowed')) {
|
||||
setTimeout(() => {
|
||||
if (isActiveRef.current) {
|
||||
setStatus('listening');
|
||||
setError(null);
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechError);
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
}, [language, conversationMode]);
|
||||
|
||||
// Update the ref when handleSpeechError changes
|
||||
useEffect(() => {
|
||||
handleSpeechErrorRef.current = handleSpeechError;
|
||||
}, [handleSpeechError]);
|
||||
|
||||
const processFinalTranscript = useCallback(async (finalText: string) => {
|
||||
if (!finalText.trim() || !isActiveRef.current) return;
|
||||
|
||||
// Prevent duplicate processing of same transcript
|
||||
if (finalText.trim() === lastTranscriptRef.current) return;
|
||||
lastTranscriptRef.current = finalText.trim();
|
||||
|
||||
// Check if provider and model are configured
|
||||
if (!currentProviderId || !currentModelId) {
|
||||
setError('No provider or model configured. Please configure them in settings.');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop listening while processing
|
||||
browserVoiceService.stopListening();
|
||||
|
||||
// Non-continuous mode: fill chat input only, do not auto-send.
|
||||
if (!conversationMode) {
|
||||
setPendingInputText(finalText.trim(), 'replace');
|
||||
processingMessageRef.current = false;
|
||||
isActiveRef.current = false;
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus('processing');
|
||||
processingMessageRef.current = true;
|
||||
|
||||
try {
|
||||
// Create session if none exists
|
||||
let sessionId = currentSessionId;
|
||||
if (!sessionId) {
|
||||
console.log('[useBrowserVoice] No active session, creating new session...');
|
||||
const newSession = await createSession();
|
||||
if (!newSession) {
|
||||
setError('Failed to create session');
|
||||
setStatus('error');
|
||||
processingMessageRef.current = false;
|
||||
return;
|
||||
}
|
||||
sessionId = newSession.id;
|
||||
console.log('[useBrowserVoice] Created new session:', sessionId);
|
||||
}
|
||||
|
||||
// Send message to AI
|
||||
await sendMessage(
|
||||
finalText.trim(),
|
||||
currentProviderId,
|
||||
currentModelId,
|
||||
currentAgentName ?? undefined
|
||||
);
|
||||
|
||||
// Wait for AI response and speak it
|
||||
// We'll poll for new assistant messages
|
||||
const checkForResponse = async () => {
|
||||
if (!isActiveRef.current) return;
|
||||
|
||||
const sessionMessages = messagesRef.current;
|
||||
const assistantMessages = Array.from(sessionMessages.values())
|
||||
.filter(m => m.info.role === 'assistant')
|
||||
.sort((a, b) => {
|
||||
const aTime = (a.info as { time?: { created?: number } }).time?.created ?? 0;
|
||||
const bTime = (b.info as { time?: { created?: number } }).time?.created ?? 0;
|
||||
return bTime - aTime;
|
||||
});
|
||||
|
||||
if (assistantMessages.length > 0) {
|
||||
const latestMessage = assistantMessages[0];
|
||||
const textParts = latestMessage.parts
|
||||
.filter(p => p.type === 'text')
|
||||
.map(p => p.text)
|
||||
.join(' ');
|
||||
|
||||
if (textParts.trim()) {
|
||||
// Speak the response
|
||||
setStatus('speaking');
|
||||
try {
|
||||
// Summarize text if enabled and over threshold
|
||||
let textToSpeak = textParts;
|
||||
if (summarizeVoiceConversation && shouldSummarize(textParts, 'voice')) {
|
||||
console.log('[useBrowserVoice] Summarizing AI response before speaking...');
|
||||
textToSpeak = await summarizeText(textParts, {
|
||||
threshold: summarizeCharacterThreshold,
|
||||
});
|
||||
} else {
|
||||
// Still sanitize for TTS even when not summarizing
|
||||
textToSpeak = sanitizeForTTS(textParts);
|
||||
}
|
||||
|
||||
// Helper to restart listening after speech ends
|
||||
// Only auto-restart if conversation mode is enabled
|
||||
const restartListening = () => {
|
||||
if (isActiveRef.current && conversationMode) {
|
||||
const isHidden = typeof document !== 'undefined' && document.visibilityState !== 'visible';
|
||||
if (isHidden) {
|
||||
pendingResumeOnVisibleRef.current = true;
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus('listening');
|
||||
if (isMobile) {
|
||||
try {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} catch (err) {
|
||||
console.error('[useBrowserVoice] Failed to restart listening:', err);
|
||||
}
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} else {
|
||||
// In non-continuous mode, return to idle after AI responds
|
||||
isActiveRef.current = false;
|
||||
setStatus('idle');
|
||||
}
|
||||
};
|
||||
|
||||
// Use server TTS when OpenAI provider is selected and available
|
||||
if (voiceProvider === 'openai' && isServerTTSAvailable) {
|
||||
console.log('[useBrowserVoice] Using OpenAI server TTS with voice:', openaiVoice);
|
||||
await speakServerTTS(textToSpeak, {
|
||||
voice: openaiVoice,
|
||||
speed: speechRate,
|
||||
onStart: () => console.log('[useBrowserVoice] Server TTS started'),
|
||||
onEnd: () => {
|
||||
console.log('[useBrowserVoice] Server TTS ended');
|
||||
restartListening();
|
||||
},
|
||||
onError: (errorMsg) => {
|
||||
console.error('[useBrowserVoice] Server TTS error:', errorMsg);
|
||||
// Show error to user when OpenAI voice fails
|
||||
setError(`OpenAI voice failed: ${errorMsg}. Please check your OpenAI API key or switch to Browser voice.`);
|
||||
setStatus('error');
|
||||
restartListening();
|
||||
}
|
||||
});
|
||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||
// Use macOS 'say' command
|
||||
console.log('[useBrowserVoice] Using macOS Say TTS with voice:', sayVoice);
|
||||
// Convert speechRate (0.5-2.0) to words per minute (100-400)
|
||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||
await speakSayTTS(textToSpeak, {
|
||||
voice: sayVoice,
|
||||
rate: wordsPerMinute,
|
||||
onStart: () => console.log('[useBrowserVoice] Say TTS started'),
|
||||
onEnd: () => {
|
||||
console.log('[useBrowserVoice] Say TTS ended');
|
||||
restartListening();
|
||||
},
|
||||
onError: (errorMsg) => {
|
||||
console.error('[useBrowserVoice] Say TTS error:', errorMsg);
|
||||
restartListening();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Use browser TTS (desktop and mobile)
|
||||
// Pre-load voices and unlock audio context before speaking
|
||||
console.log('[useBrowserVoice] Using browser TTS');
|
||||
|
||||
// Warn user if they selected OpenAI but it's unavailable
|
||||
if (voiceProvider === 'openai' && !isServerTTSAvailable) {
|
||||
console.warn('[useBrowserVoice] OpenAI voice selected but unavailable, falling back to browser voice');
|
||||
setError('OpenAI voice unavailable (API key not configured). Using browser voice instead.');
|
||||
}
|
||||
|
||||
await browserVoiceService.waitForVoices();
|
||||
await browserVoiceService.resumeAudioContext();
|
||||
|
||||
await browserVoiceService.speakText(textToSpeak, language, () => {
|
||||
// When speech ends, go back to listening if still active
|
||||
restartListening();
|
||||
}, { rate: speechRate, pitch: speechPitch, volume: speechVolume, voiceName: browserVoice || undefined });
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Speech failed';
|
||||
|
||||
// Ignore errors if we've stopped voice (e.g., user cancelled during speech)
|
||||
if (!isActiveRef.current) {
|
||||
console.log('[useBrowserVoice] Ignoring speech error after voice stopped:', errorMsg);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error('[useBrowserVoice] Speech error:', errorMsg);
|
||||
|
||||
// Check for autoplay policy error
|
||||
if (errorMsg.includes('not-allowed') || errorMsg.includes('autoplay')) {
|
||||
setError('Audio blocked by browser. Please click the voice button again to enable audio.');
|
||||
}
|
||||
|
||||
// Only restart listening if conversation mode is enabled
|
||||
if (conversationMode) {
|
||||
setStatus('listening');
|
||||
if (isMobile) {
|
||||
try {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} catch (restartErr) {
|
||||
console.error('[useBrowserVoice] Failed to restart listening after speech error:', restartErr);
|
||||
}
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} else {
|
||||
// In non-continuous mode, return to idle after error
|
||||
isActiveRef.current = false;
|
||||
setStatus('idle');
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Check again in 500ms
|
||||
setTimeout(checkForResponse, 500);
|
||||
};
|
||||
|
||||
// Start checking for response after a short delay
|
||||
setTimeout(checkForResponse, 1000);
|
||||
|
||||
} catch (err) {
|
||||
console.error('[useBrowserVoice] Send message error:', err);
|
||||
setError(err instanceof Error ? err.message : 'Failed to send message');
|
||||
setStatus('error');
|
||||
processingMessageRef.current = false;
|
||||
}
|
||||
}, [currentSessionId, currentProviderId, currentModelId, currentAgentName, language, sendMessage, setPendingInputText, createSession, speechRate, speechPitch, speechVolume, isMobile, isServerTTSAvailable, speakServerTTS, isSayTTSAvailable, speakSayTTS, voiceProvider, sayVoice, browserVoice, openaiVoice, summarizeVoiceConversation, summarizeCharacterThreshold, conversationMode]);
|
||||
|
||||
// Handle speech recognition result
|
||||
const handleSpeechResult = useCallback(async (text: string, isFinal: boolean) => {
|
||||
if (!isActiveRef.current) return;
|
||||
const normalized = text.trim();
|
||||
if (!isFinal || !normalized) return;
|
||||
|
||||
pendingFinalTranscriptRef.current = normalized;
|
||||
|
||||
if (finalTranscriptTimerRef.current) {
|
||||
clearTimeout(finalTranscriptTimerRef.current);
|
||||
}
|
||||
|
||||
finalTranscriptTimerRef.current = setTimeout(() => {
|
||||
finalTranscriptTimerRef.current = null;
|
||||
const transcript = pendingFinalTranscriptRef.current.trim();
|
||||
pendingFinalTranscriptRef.current = '';
|
||||
if (!transcript) return;
|
||||
void processFinalTranscript(transcript);
|
||||
}, FINAL_TRANSCRIPT_SETTLE_MS);
|
||||
}, [processFinalTranscript]);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof document === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleVisibilityChange = () => {
|
||||
if (document.visibilityState !== 'visible') {
|
||||
return;
|
||||
}
|
||||
if (!pendingResumeOnVisibleRef.current) {
|
||||
return;
|
||||
}
|
||||
if (!isActiveRef.current || !conversationMode) {
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
setStatus('listening');
|
||||
try {
|
||||
if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to resume voice';
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener('visibilitychange', handleVisibilityChange);
|
||||
return () => {
|
||||
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
||||
};
|
||||
}, [conversationMode, isMobile, language]);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof navigator === 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const mediaDevices = navigator.mediaDevices;
|
||||
if (!mediaDevices || typeof mediaDevices.addEventListener !== 'function') {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleDeviceChange = () => {
|
||||
if (!isActiveRef.current || status !== 'listening') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
}
|
||||
|
||||
deviceChangeRestartTimerRef.current = setTimeout(() => {
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
if (!isActiveRef.current || status !== 'listening') {
|
||||
return;
|
||||
}
|
||||
|
||||
const isHidden = typeof document !== 'undefined' && document.visibilityState !== 'visible';
|
||||
if (isHidden) {
|
||||
pendingResumeOnVisibleRef.current = true;
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
browserVoiceService.stopListening();
|
||||
if (isMobile) {
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
} else {
|
||||
void browserVoiceService.startListening(language, handleSpeechResultRef.current!, handleSpeechErrorRef.current!);
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Microphone source changed. Tap mic to continue.';
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
}, DEVICE_CHANGE_RESTART_DELAY_MS);
|
||||
};
|
||||
|
||||
mediaDevices.addEventListener('devicechange', handleDeviceChange);
|
||||
|
||||
return () => {
|
||||
mediaDevices.removeEventListener('devicechange', handleDeviceChange);
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [isMobile, language, status]);
|
||||
|
||||
// Update the ref when handleSpeechResult changes
|
||||
useEffect(() => {
|
||||
handleSpeechResultRef.current = handleSpeechResult;
|
||||
}, [handleSpeechResult]);
|
||||
|
||||
// Prepare voice for mobile (request permission)
|
||||
const prepareVoice = useCallback(async (): Promise<boolean> => {
|
||||
if (!isSupported) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await browserVoiceService.prepareListening();
|
||||
return true;
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Microphone permission denied';
|
||||
setError(errorMsg);
|
||||
return false;
|
||||
}
|
||||
}, [isSupported]);
|
||||
|
||||
// Start voice mode
|
||||
const startVoice = useCallback(async () => {
|
||||
if (!isSupported) {
|
||||
setError('Browser voice not supported');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!currentSessionId) {
|
||||
setError('No active session');
|
||||
setStatus('error');
|
||||
return;
|
||||
}
|
||||
|
||||
isActiveRef.current = true;
|
||||
lastTranscriptRef.current = '';
|
||||
setError(null);
|
||||
setStatus('listening');
|
||||
|
||||
// On mobile, use sync path to ensure SpeechRecognition.start() is called
|
||||
// within the same user gesture context (required by iOS Safari)
|
||||
// Also unlock audio immediately for TTS playback later
|
||||
if (isMobile) {
|
||||
try {
|
||||
// Unlock audio context synchronously within user gesture
|
||||
browserVoiceService.unlockAudio().catch(() => {
|
||||
// Audio unlock failed, but continue anyway
|
||||
});
|
||||
// Also unlock server TTS audio for mobile Safari (OpenAI)
|
||||
unlockServerTTSAudio().catch(() => {
|
||||
// Server TTS unlock failed, but continue anyway
|
||||
});
|
||||
// Also unlock Say TTS audio for mobile Safari (macOS Say)
|
||||
unlockSayTTSAudio().catch(() => {
|
||||
// Say TTS unlock failed, but continue anyway
|
||||
});
|
||||
browserVoiceService.startListeningSync(language, handleSpeechResult, handleSpeechError);
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
|
||||
console.error('[useBrowserVoice] Mobile voice start error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
} else {
|
||||
// Desktop can use async path with permission check
|
||||
try {
|
||||
await browserVoiceService.startListening(language, handleSpeechResult, handleSpeechError);
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to start voice';
|
||||
console.error('[useBrowserVoice] Desktop voice start error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
setStatus('error');
|
||||
isActiveRef.current = false;
|
||||
}
|
||||
}
|
||||
}, [isSupported, currentSessionId, language, handleSpeechResult, handleSpeechError, isMobile, unlockServerTTSAudio, unlockSayTTSAudio]);
|
||||
|
||||
// Stop voice mode
|
||||
const stopVoice = useCallback(() => {
|
||||
isActiveRef.current = false;
|
||||
processingMessageRef.current = false;
|
||||
pendingResumeOnVisibleRef.current = false;
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
pendingFinalTranscriptRef.current = '';
|
||||
if (finalTranscriptTimerRef.current) {
|
||||
clearTimeout(finalTranscriptTimerRef.current);
|
||||
finalTranscriptTimerRef.current = null;
|
||||
}
|
||||
browserVoiceService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
stopServerTTS(); // Also stop server TTS if playing
|
||||
stopSayTTS(); // Also stop Say TTS if playing
|
||||
setStatus('idle');
|
||||
setError(null);
|
||||
}, [stopServerTTS, stopSayTTS]);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
isActiveRef.current = false;
|
||||
if (deviceChangeRestartTimerRef.current) {
|
||||
clearTimeout(deviceChangeRestartTimerRef.current);
|
||||
deviceChangeRestartTimerRef.current = null;
|
||||
}
|
||||
pendingFinalTranscriptRef.current = '';
|
||||
if (finalTranscriptTimerRef.current) {
|
||||
clearTimeout(finalTranscriptTimerRef.current);
|
||||
finalTranscriptTimerRef.current = null;
|
||||
}
|
||||
browserVoiceService.setConversationMode(false);
|
||||
browserVoiceService.stopListening();
|
||||
browserVoiceService.cancelSpeech();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
status,
|
||||
isSupported,
|
||||
error,
|
||||
language,
|
||||
setLanguage,
|
||||
startVoice,
|
||||
stopVoice,
|
||||
conversationMode,
|
||||
toggleConversationMode,
|
||||
prepareVoice,
|
||||
isMobile,
|
||||
voiceProvider,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* useMessageTTS Hook
|
||||
*
|
||||
* Hook for playing TTS on individual messages.
|
||||
* Uses the configured voice provider (browser, OpenAI, or macOS Say).
|
||||
*/
|
||||
|
||||
import { useCallback, useState } from 'react';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useServerTTS } from './useServerTTS';
|
||||
import { useSayTTS } from './useSayTTS';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { summarizeText, shouldSummarize, sanitizeForTTS } from '@/lib/voice/summarize';
|
||||
|
||||
export interface UseMessageTTSReturn {
|
||||
/** Whether TTS is currently playing for this message */
|
||||
isPlaying: boolean;
|
||||
/** Play the message text */
|
||||
play: (text: string) => Promise<void>;
|
||||
/** Stop playback */
|
||||
stop: () => void;
|
||||
}
|
||||
|
||||
export function useMessageTTS(): UseMessageTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
|
||||
const {
|
||||
voiceProvider,
|
||||
speechRate,
|
||||
speechPitch,
|
||||
speechVolume,
|
||||
sayVoice,
|
||||
browserVoice,
|
||||
openaiVoice,
|
||||
summarizeMessageTTS,
|
||||
summarizeCharacterThreshold,
|
||||
} = useConfigStore();
|
||||
|
||||
const { speak: speakServerTTS, stop: stopServerTTS, isAvailable: isServerTTSAvailable } = useServerTTS();
|
||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS();
|
||||
|
||||
const stop = useCallback(() => {
|
||||
setIsPlaying(false);
|
||||
stopServerTTS();
|
||||
stopSayTTS();
|
||||
browserVoiceService.cancelSpeech();
|
||||
}, [stopServerTTS, stopSayTTS]);
|
||||
|
||||
const play = useCallback(async (text: string) => {
|
||||
if (!text.trim()) return;
|
||||
|
||||
// Stop any existing playback
|
||||
stop();
|
||||
|
||||
setIsPlaying(true);
|
||||
|
||||
try {
|
||||
// Summarize text if enabled and over threshold
|
||||
let textToSpeak = text;
|
||||
if (summarizeMessageTTS && shouldSummarize(text, 'message')) {
|
||||
textToSpeak = await summarizeText(text, {
|
||||
threshold: summarizeCharacterThreshold,
|
||||
});
|
||||
} else {
|
||||
// Still sanitize for TTS even when not summarizing
|
||||
textToSpeak = sanitizeForTTS(text);
|
||||
}
|
||||
|
||||
if (voiceProvider === 'openai' && isServerTTSAvailable) {
|
||||
await speakServerTTS(textToSpeak, {
|
||||
voice: openaiVoice,
|
||||
speed: speechRate,
|
||||
summarize: false, // We already summarized client-side
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||
await speakSayTTS(textToSpeak, {
|
||||
voice: sayVoice,
|
||||
rate: wordsPerMinute,
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else {
|
||||
// Browser TTS
|
||||
await browserVoiceService.waitForVoices();
|
||||
await browserVoiceService.resumeAudioContext();
|
||||
await browserVoiceService.speakText(
|
||||
textToSpeak,
|
||||
navigator.language || 'en-US',
|
||||
() => setIsPlaying(false),
|
||||
{
|
||||
rate: speechRate,
|
||||
pitch: speechPitch,
|
||||
volume: speechVolume,
|
||||
voiceName: browserVoice || undefined,
|
||||
}
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[useMessageTTS] Playback error:', err);
|
||||
setIsPlaying(false);
|
||||
}
|
||||
}, [
|
||||
voiceProvider,
|
||||
speechRate,
|
||||
speechPitch,
|
||||
speechVolume,
|
||||
sayVoice,
|
||||
browserVoice,
|
||||
openaiVoice,
|
||||
summarizeMessageTTS,
|
||||
summarizeCharacterThreshold,
|
||||
isServerTTSAvailable,
|
||||
isSayTTSAvailable,
|
||||
speakServerTTS,
|
||||
speakSayTTS,
|
||||
stop,
|
||||
]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
play,
|
||||
stop,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
/**
|
||||
* useSayTTS Hook
|
||||
*
|
||||
* React hook for macOS 'say' command text-to-speech playback.
|
||||
* Uses the native macOS speech synthesis via server API.
|
||||
* Uses Web Audio API for playback (better iOS Safari support).
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const { speak, isPlaying, stop, isAvailable } = useSayTTS();
|
||||
*
|
||||
* // Speak text
|
||||
* await speak('Hello, this is a test');
|
||||
*
|
||||
* // Stop playback
|
||||
* stop();
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
export interface UseSayTTSReturn {
|
||||
/** Whether TTS is currently playing */
|
||||
isPlaying: boolean;
|
||||
/** Whether the macOS say command is available */
|
||||
isAvailable: boolean;
|
||||
/** Available voices */
|
||||
voices: Array<{ name: string; locale: string }>;
|
||||
/** Current error if any */
|
||||
error: string | null;
|
||||
/** Speak the given text */
|
||||
speak: (text: string, options?: SpeakOptions) => Promise<void>;
|
||||
/** Stop current playback */
|
||||
stop: () => void;
|
||||
/** Check if service is available */
|
||||
checkAvailability: () => Promise<boolean>;
|
||||
/** Unlock audio for mobile Safari - call this on user gesture */
|
||||
unlockAudio: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface SpeakOptions {
|
||||
/** Voice to use (defaults to Samantha) */
|
||||
voice?: string;
|
||||
/** Speech rate in words per minute (defaults to 200) */
|
||||
rate?: number;
|
||||
/** Callback when playback starts */
|
||||
onStart?: () => void;
|
||||
/** Callback when playback ends */
|
||||
onEnd?: () => void;
|
||||
/** Callback on error */
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
// Shared AudioContext for Web Audio API playback (better iOS support)
|
||||
let sharedAudioContext: AudioContext | null = null;
|
||||
|
||||
function getAudioContext(): AudioContext {
|
||||
if (!sharedAudioContext) {
|
||||
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
return sharedAudioContext;
|
||||
}
|
||||
|
||||
export function useSayTTS(): UseSayTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isAvailable, setIsAvailable] = useState(false);
|
||||
const [voices, setVoices] = useState<Array<{ name: string; locale: string }>>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
|
||||
// Unlock audio for mobile Safari - must be called within user gesture
|
||||
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||
try {
|
||||
// Get or create AudioContext
|
||||
const ctx = getAudioContext();
|
||||
|
||||
// Resume if suspended (required for iOS Safari)
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useSayTTS] AudioContext resumed');
|
||||
}
|
||||
|
||||
// Play a tiny silent buffer to fully unlock
|
||||
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(ctx.destination);
|
||||
source.start(0);
|
||||
|
||||
console.log('[useSayTTS] Audio unlocked for mobile playback');
|
||||
} catch (err) {
|
||||
console.error('[useSayTTS] Failed to unlock audio:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Check if macOS say is available
|
||||
const checkAvailability = useCallback(async (): Promise<boolean> => {
|
||||
try {
|
||||
const response = await fetch('/api/tts/say/status');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setIsAvailable(data.available);
|
||||
if (data.voices) {
|
||||
setVoices(data.voices);
|
||||
}
|
||||
return data.available;
|
||||
}
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
} catch (err) {
|
||||
console.error('[useSayTTS] Failed to check availability:', err);
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Check availability on mount
|
||||
useEffect(() => {
|
||||
checkAvailability();
|
||||
}, [checkAvailability]);
|
||||
|
||||
// Stop current playback
|
||||
const stop = useCallback(() => {
|
||||
if (audioSourceRef.current) {
|
||||
try {
|
||||
audioSourceRef.current.stop();
|
||||
} catch {
|
||||
// Already stopped
|
||||
}
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
}, []);
|
||||
|
||||
// Speak text using macOS say
|
||||
const speak = useCallback(async (text: string, options?: SpeakOptions): Promise<void> => {
|
||||
// Stop any existing playback
|
||||
stop();
|
||||
|
||||
if (!text.trim()) {
|
||||
setError('No text to speak');
|
||||
options?.onError?.('No text to speak');
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Create abort controller for this request
|
||||
abortControllerRef.current = new AbortController();
|
||||
|
||||
// Fetch audio from server
|
||||
const response = await fetch('/api/tts/say/speak', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text: text.trim(),
|
||||
voice: options?.voice || 'Samantha',
|
||||
rate: options?.rate || 200,
|
||||
}),
|
||||
signal: abortControllerRef.current.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
// Get audio data from response
|
||||
const audioBlob = await response.blob();
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
console.log('[useSayTTS] Received audio:', audioBlob.size, 'bytes');
|
||||
|
||||
// Use Web Audio API for playback (same as useServerTTS)
|
||||
const ctx = getAudioContext();
|
||||
|
||||
// Resume context if suspended
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useSayTTS] AudioContext resumed before playback');
|
||||
}
|
||||
|
||||
// Decode audio data
|
||||
console.log('[useSayTTS] Decoding audio data...');
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
|
||||
// Create source node
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
audioSourceRef.current = source;
|
||||
|
||||
// Set up event handlers
|
||||
source.onended = () => {
|
||||
console.log('[useSayTTS] Audio playback ended');
|
||||
setIsPlaying(false);
|
||||
audioSourceRef.current = null;
|
||||
options?.onEnd?.();
|
||||
};
|
||||
|
||||
// Start playback
|
||||
console.log('[useSayTTS] Starting audio playback via Web Audio API...');
|
||||
setIsPlaying(true);
|
||||
options?.onStart?.();
|
||||
source.start(0);
|
||||
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||
console.error('[useSayTTS] Error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
options?.onError?.(errorMsg);
|
||||
setIsPlaying(false);
|
||||
}
|
||||
}, [stop]);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
isAvailable,
|
||||
voices,
|
||||
error,
|
||||
speak,
|
||||
stop,
|
||||
checkAvailability,
|
||||
unlockAudio,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* useServerTTS Hook
|
||||
*
|
||||
* React hook for server-side text-to-speech playback.
|
||||
* Fetches audio from the server and plays it, bypassing mobile Safari restrictions.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const { speak, isPlaying, stop, isAvailable } = useServerTTS();
|
||||
*
|
||||
* // Speak text
|
||||
* await speak('Hello, this is a test');
|
||||
*
|
||||
* // Stop playback
|
||||
* stop();
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
export interface UseServerTTSReturn {
|
||||
/** Whether TTS is currently playing */
|
||||
isPlaying: boolean;
|
||||
/** Whether the server TTS service is available */
|
||||
isAvailable: boolean;
|
||||
/** Current error if any */
|
||||
error: string | null;
|
||||
/** Speak the given text */
|
||||
speak: (text: string, options?: SpeakOptions) => Promise<void>;
|
||||
/** Stop current playback */
|
||||
stop: () => void;
|
||||
/** Check if service is available */
|
||||
checkAvailability: () => Promise<boolean>;
|
||||
/** Unlock audio for mobile Safari - call this on user gesture before speaking */
|
||||
unlockAudio: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface SpeakOptions {
|
||||
/** Voice to use (defaults to coral) */
|
||||
voice?: string;
|
||||
/** Speech speed (0.25 to 4.0, defaults to 1.0) */
|
||||
speed?: number;
|
||||
/** Optional instructions for the voice */
|
||||
instructions?: string;
|
||||
/** Summarize long text before speaking (defaults to true) */
|
||||
summarize?: boolean;
|
||||
/** Provider ID for summarization model */
|
||||
providerId?: string;
|
||||
/** Model ID for summarization */
|
||||
modelId?: string;
|
||||
/** Character threshold for summarization (defaults to 200) */
|
||||
threshold?: number;
|
||||
/** Callback when playback starts */
|
||||
onStart?: () => void;
|
||||
/** Callback when playback ends */
|
||||
onEnd?: () => void;
|
||||
/** Callback on error */
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
// Shared AudioContext for Web Audio API playback (better iOS support)
|
||||
let sharedAudioContext: AudioContext | null = null;
|
||||
|
||||
function getAudioContext(): AudioContext {
|
||||
if (!sharedAudioContext) {
|
||||
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
return sharedAudioContext;
|
||||
}
|
||||
|
||||
export function useServerTTS(): UseServerTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isAvailable, setIsAvailable] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
|
||||
// Get current model, threshold, and max length from config store for summarization
|
||||
const { currentProviderId, currentModelId, summarizeCharacterThreshold, summarizeMaxLength, openaiApiKey } = useConfigStore();
|
||||
|
||||
// Check if server TTS is available
|
||||
const checkAvailability = useCallback(async (): Promise<boolean> => {
|
||||
try {
|
||||
const response = await fetch('/api/tts/status');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
// Available if server has key OR user has provided their own key
|
||||
const hasServerKey = data.available;
|
||||
const hasClientKey = openaiApiKey && openaiApiKey.trim().length > 0;
|
||||
const available = hasServerKey || hasClientKey;
|
||||
setIsAvailable(available);
|
||||
return available;
|
||||
}
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
} catch (err) {
|
||||
console.error('[useServerTTS] Failed to check availability:', err);
|
||||
setIsAvailable(false);
|
||||
return false;
|
||||
}
|
||||
}, [openaiApiKey]);
|
||||
|
||||
// Check availability on mount and when API key changes
|
||||
useEffect(() => {
|
||||
checkAvailability();
|
||||
}, [checkAvailability]);
|
||||
|
||||
// Stop current playback
|
||||
const stop = useCallback(() => {
|
||||
// Stop Web Audio API source
|
||||
if (audioSourceRef.current) {
|
||||
try {
|
||||
audioSourceRef.current.stop();
|
||||
} catch {
|
||||
// Already stopped
|
||||
}
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
}, []);
|
||||
|
||||
// Pre-unlock audio for mobile Safari
|
||||
// This must be called within a user gesture context
|
||||
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||
try {
|
||||
// Get or create AudioContext
|
||||
const ctx = getAudioContext();
|
||||
|
||||
// Resume if suspended (required for iOS Safari)
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useServerTTS] AudioContext resumed');
|
||||
}
|
||||
|
||||
// Play a tiny silent buffer to fully unlock
|
||||
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(ctx.destination);
|
||||
source.start(0);
|
||||
|
||||
console.log('[useServerTTS] Audio unlocked for mobile playback');
|
||||
} catch (err) {
|
||||
console.error('[useServerTTS] Failed to unlock audio:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Speak text using server TTS
|
||||
const speak = useCallback(async (text: string, options?: SpeakOptions): Promise<void> => {
|
||||
// Stop any existing playback
|
||||
stop();
|
||||
|
||||
if (!text.trim()) {
|
||||
setError('No text to speak');
|
||||
options?.onError?.('No text to speak');
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Unlock audio context first (required for mobile Safari)
|
||||
// Must be done before any async operations to stay within user gesture context
|
||||
const ctx = getAudioContext();
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
console.log('[useServerTTS] AudioContext resumed');
|
||||
}
|
||||
|
||||
// Play a silent buffer to fully unlock audio on iOS
|
||||
const silentBuffer = ctx.createBuffer(1, 1, 22050);
|
||||
const silentSource = ctx.createBufferSource();
|
||||
silentSource.buffer = silentBuffer;
|
||||
silentSource.connect(ctx.destination);
|
||||
silentSource.start(0);
|
||||
|
||||
// Create abort controller for this request
|
||||
abortControllerRef.current = new AbortController();
|
||||
|
||||
const voice = options?.voice || 'nova';
|
||||
console.log('[useServerTTS] Speaking with voice:', voice, 'options:', options);
|
||||
|
||||
// Fetch audio from server
|
||||
const response = await fetch('/api/tts/speak', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text: text.trim(),
|
||||
voice,
|
||||
speed: options?.speed || 0.9,
|
||||
instructions: options?.instructions,
|
||||
summarize: options?.summarize ?? true, // Summarize by default for voice output
|
||||
// Use provided provider/model, or fall back to current chat model
|
||||
providerId: options?.providerId || currentProviderId || undefined,
|
||||
modelId: options?.modelId || currentModelId || undefined,
|
||||
// Use provided threshold, or fall back to user setting, or default to 200
|
||||
threshold: options?.threshold ?? summarizeCharacterThreshold ?? 200,
|
||||
// Max character length for summaries
|
||||
maxLength: summarizeMaxLength ?? 500,
|
||||
// Send API key from settings if available
|
||||
apiKey: openaiApiKey || undefined,
|
||||
}),
|
||||
signal: abortControllerRef.current.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
// Get audio data from response
|
||||
const audioBlob = await response.blob();
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
|
||||
// Decode audio data using the same context we unlocked earlier
|
||||
console.log('[useServerTTS] Decoding audio data...');
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
|
||||
// Create source node
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
audioSourceRef.current = source;
|
||||
|
||||
// Set up event handlers
|
||||
source.onended = () => {
|
||||
console.log('[useServerTTS] Audio playback ended');
|
||||
setIsPlaying(false);
|
||||
audioSourceRef.current = null;
|
||||
options?.onEnd?.();
|
||||
};
|
||||
|
||||
// Start playback
|
||||
console.log('[useServerTTS] Starting audio playback via Web Audio API...');
|
||||
setIsPlaying(true);
|
||||
options?.onStart?.();
|
||||
source.start(0);
|
||||
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError') {
|
||||
// Request was aborted, don't show error
|
||||
return;
|
||||
}
|
||||
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||
console.error('[useServerTTS] Error:', errorMsg);
|
||||
setError(errorMsg);
|
||||
options?.onError?.(errorMsg);
|
||||
setIsPlaying(false);
|
||||
}
|
||||
}, [stop, currentProviderId, currentModelId, summarizeCharacterThreshold, summarizeMaxLength, openaiApiKey]);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
isAvailable,
|
||||
error,
|
||||
speak,
|
||||
stop,
|
||||
checkAvailability,
|
||||
unlockAudio,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useSessionStore } from '@/stores/useSessionStore';
|
||||
import { voiceHooks, isVoiceSessionStarted } from '@/lib/voice';
|
||||
|
||||
/**
|
||||
* Hook that syncs session events (messages, permissions) to the voice agent.
|
||||
* Call this inside VoiceProvider to enable session awareness during voice.
|
||||
*/
|
||||
export function useVoiceContext() {
|
||||
const currentSessionId = useSessionStore((s) => s.currentSessionId);
|
||||
const messages = useSessionStore((s) =>
|
||||
currentSessionId ? s.messages.get(currentSessionId) : undefined
|
||||
);
|
||||
const permissions = useSessionStore((s) =>
|
||||
currentSessionId ? s.permissions.get(currentSessionId) : undefined
|
||||
);
|
||||
|
||||
// Track last seen message count to only forward new messages
|
||||
const lastMessageCountRef = useRef(0);
|
||||
|
||||
// Forward new messages to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !messages || !isVoiceSessionStarted()) return;
|
||||
|
||||
const currentCount = messages.length;
|
||||
if (currentCount <= lastMessageCountRef.current) return;
|
||||
|
||||
// Get only new messages (messages since last check)
|
||||
const newMessages = messages.slice(lastMessageCountRef.current);
|
||||
lastMessageCountRef.current = currentCount;
|
||||
|
||||
// Format for voice hooks (extract role and content)
|
||||
const formattedMessages = newMessages.map(m => ({
|
||||
role: m.info.role,
|
||||
content: m.parts.map(p => ('text' in p ? p.text : '')).join('')
|
||||
}));
|
||||
|
||||
voiceHooks.onMessages(currentSessionId, formattedMessages);
|
||||
}, [currentSessionId, messages]);
|
||||
|
||||
// Forward permission requests to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !permissions || permissions.length === 0) return;
|
||||
if (!isVoiceSessionStarted()) return;
|
||||
|
||||
const request = permissions[0];
|
||||
if (!request) return;
|
||||
|
||||
voiceHooks.onPermissionRequested(
|
||||
currentSessionId,
|
||||
request.id,
|
||||
request.permission,
|
||||
request.metadata
|
||||
);
|
||||
}, [currentSessionId, permissions]);
|
||||
|
||||
// Reset message count when session changes
|
||||
useEffect(() => {
|
||||
lastMessageCountRef.current = 0;
|
||||
}, [currentSessionId]);
|
||||
}
|
||||
@@ -0,0 +1,654 @@
|
||||
/**
|
||||
* Browser Voice Service - Web Speech API wrapper
|
||||
*
|
||||
* Provides speech recognition (STT) and speech synthesis (TTS) using
|
||||
* browser-native Web Speech API. No external dependencies required.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { browserVoiceService } from './browserVoiceService';
|
||||
*
|
||||
* // Check support
|
||||
* if (browserVoiceService.isSupported()) {
|
||||
* // Start listening
|
||||
* browserVoiceService.startListening('en-US', (text, isFinal) => {
|
||||
* if (isFinal) {
|
||||
* console.log('Final transcript:', text);
|
||||
* }
|
||||
* });
|
||||
*
|
||||
* // Speak text
|
||||
* browserVoiceService.speakText('Hello world', 'en-US', () => {
|
||||
* console.log('Speech finished');
|
||||
* });
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
|
||||
// Extend Window interface for SpeechRecognition
|
||||
declare global {
|
||||
interface Window {
|
||||
SpeechRecognition: { new(): SpeechRecognition };
|
||||
webkitSpeechRecognition: { new(): SpeechRecognition };
|
||||
}
|
||||
}
|
||||
|
||||
// Callback types
|
||||
export type SpeechResultCallback = (text: string, isFinal: boolean) => void;
|
||||
export type SpeechEndCallback = () => void;
|
||||
export type ErrorCallback = (error: string) => void;
|
||||
|
||||
/**
|
||||
* Browser Voice Service class
|
||||
* Wraps Web Speech API with a clean interface
|
||||
*/
|
||||
class BrowserVoiceService {
|
||||
private recognition: SpeechRecognition | null = null;
|
||||
private isListening = false;
|
||||
private currentLang = 'en-US';
|
||||
private onResultCallback: SpeechResultCallback | null = null;
|
||||
private onErrorCallback: ErrorCallback | null = null;
|
||||
private restartOnEnd = false;
|
||||
private conversationMode = false;
|
||||
private isSpeaking = false;
|
||||
private audioContext: AudioContext | null = null;
|
||||
private audioUnlockRequired = false;
|
||||
|
||||
/**
|
||||
* Check if browser supports Web Speech API
|
||||
*/
|
||||
isSupported(): boolean {
|
||||
const hasRecognition = 'SpeechRecognition' in window || 'webkitSpeechRecognition' in window;
|
||||
const hasSynthesis = 'speechSynthesis' in window;
|
||||
return hasRecognition && hasSynthesis;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get detailed support information
|
||||
*/
|
||||
getSupportDetails(): {
|
||||
recognition: boolean;
|
||||
synthesis: boolean;
|
||||
prefixed: boolean;
|
||||
secureContext: boolean;
|
||||
} {
|
||||
return {
|
||||
recognition: 'SpeechRecognition' in window || 'webkitSpeechRecognition' in window,
|
||||
synthesis: 'speechSynthesis' in window,
|
||||
prefixed: !('SpeechRecognition' in window) && 'webkitSpeechRecognition' in window,
|
||||
secureContext: window.isSecureContext,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Set conversation mode (continuous back-and-forth)
|
||||
* @param enabled - Whether to enable continuous conversation mode
|
||||
*
|
||||
* Note: This only sets the flag - it does NOT auto-start listening.
|
||||
* The user must explicitly start voice mode first. Conversation mode
|
||||
* only affects whether listening auto-resumes after AI finishes speaking.
|
||||
*/
|
||||
setConversationMode(enabled: boolean): void {
|
||||
this.conversationMode = enabled;
|
||||
// If disabling, stop auto-restart
|
||||
if (!enabled) {
|
||||
this.restartOnEnd = false;
|
||||
}
|
||||
// Note: We intentionally do NOT auto-start listening here.
|
||||
// The user must explicitly click the microphone button to start.
|
||||
// Conversation mode only controls auto-resume after speech ends.
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if conversation mode is active
|
||||
*/
|
||||
isConversationMode(): boolean {
|
||||
return this.conversationMode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause listening temporarily (e.g., while AI is speaking)
|
||||
*/
|
||||
pauseListening(): void {
|
||||
this.restartOnEnd = false;
|
||||
if (this.recognition) {
|
||||
try {
|
||||
this.recognition.stop();
|
||||
} catch {
|
||||
// Ignore stop errors
|
||||
}
|
||||
}
|
||||
this.isListening = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume listening after being paused
|
||||
*/
|
||||
resumeListening(): void {
|
||||
console.log('[BrowserVoiceService] resumeListening called:', {
|
||||
conversationMode: this.conversationMode,
|
||||
hasCallback: !!this.onResultCallback,
|
||||
isSpeaking: this.isSpeaking,
|
||||
currentLang: this.currentLang
|
||||
});
|
||||
|
||||
if (this.conversationMode && this.onResultCallback && !this.isSpeaking) {
|
||||
// Use sync version for resume (should already have permission)
|
||||
try {
|
||||
console.log('[BrowserVoiceService] Resuming listening...');
|
||||
this.startListeningSync(this.currentLang, this.onResultCallback, this.onErrorCallback || undefined);
|
||||
} catch (err) {
|
||||
console.error('[BrowserVoiceService] Failed to resume listening:', err);
|
||||
}
|
||||
} else {
|
||||
console.log('[BrowserVoiceService] Not resuming - conditions not met');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if microphone permission is already granted
|
||||
* @returns Promise<boolean> - true if permission granted
|
||||
*/
|
||||
async checkMicrophonePermission(): Promise<boolean> {
|
||||
try {
|
||||
if ('permissions' in navigator) {
|
||||
const result = await navigator.permissions.query({ name: 'microphone' as PermissionName });
|
||||
return result.state === 'granted';
|
||||
}
|
||||
return false;
|
||||
} catch {
|
||||
// permissions API not supported or failed
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare listening by requesting microphone permission
|
||||
* This should be called BEFORE user gesture on mobile to pre-request permission
|
||||
* @returns Promise<boolean> - true if permission granted
|
||||
*/
|
||||
async prepareListening(): Promise<boolean> {
|
||||
if (!this.isSupported()) {
|
||||
throw new Error('Web Speech API not supported in this browser');
|
||||
}
|
||||
|
||||
if (typeof navigator === 'undefined' || typeof navigator.mediaDevices?.getUserMedia !== 'function') {
|
||||
// Some embedded runtimes (e.g. desktop webviews) may not expose mediaDevices,
|
||||
// while SpeechRecognition can still request mic permission on start().
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
return true;
|
||||
} catch (err) {
|
||||
const name = typeof err === 'object' && err && 'name' in err ? String((err as { name?: unknown }).name) : '';
|
||||
if (name === 'NotAllowedError') {
|
||||
throw new Error('Microphone permission denied');
|
||||
}
|
||||
if (name === 'NotFoundError') {
|
||||
throw new Error('No microphone found');
|
||||
}
|
||||
const errorMsg = err instanceof Error ? err.message : 'Unable to access microphone';
|
||||
throw new Error(errorMsg);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start speech recognition SYNCHRONOUSLY
|
||||
* Must be called within a user gesture handler on mobile (iOS Safari)
|
||||
* @param lang - BCP 47 language tag (e.g., 'en-US', 'es-ES')
|
||||
* @param onResult - Callback for speech results
|
||||
* @param onError - Optional callback for errors
|
||||
*/
|
||||
startListeningSync(
|
||||
lang: string,
|
||||
onResult: SpeechResultCallback,
|
||||
onError?: ErrorCallback
|
||||
): void {
|
||||
if (!this.isSupported()) {
|
||||
const errorMsg = 'Web Speech API not supported in this browser';
|
||||
onError?.(errorMsg);
|
||||
throw new Error(errorMsg);
|
||||
}
|
||||
|
||||
// Stop any existing recognition
|
||||
this.stopListening();
|
||||
|
||||
// Create new recognition instance
|
||||
const SpeechRecognitionConstructor = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||
this.recognition = new SpeechRecognitionConstructor();
|
||||
this.currentLang = lang;
|
||||
this.onResultCallback = onResult;
|
||||
this.onErrorCallback = onError || null;
|
||||
|
||||
// Configure recognition
|
||||
this.recognition.continuous = true;
|
||||
this.recognition.interimResults = true;
|
||||
this.recognition.lang = lang;
|
||||
|
||||
// Set up event handlers
|
||||
this.recognition.onstart = () => {
|
||||
console.log('[BrowserVoiceService] Recognition started');
|
||||
this.isListening = true;
|
||||
this.restartOnEnd = true;
|
||||
};
|
||||
|
||||
this.recognition.onaudiostart = () => {
|
||||
console.log('[BrowserVoiceService] Audio recording started');
|
||||
};
|
||||
|
||||
this.recognition.onsoundstart = () => {
|
||||
console.log('[BrowserVoiceService] Sound detected');
|
||||
};
|
||||
|
||||
this.recognition.onresult = (event: SpeechRecognitionEvent) => {
|
||||
console.log('[BrowserVoiceService] Got result:', event.results.length, 'results');
|
||||
let finalTranscript = '';
|
||||
let interimTranscript = '';
|
||||
|
||||
for (let i = event.resultIndex; i < event.results.length; i++) {
|
||||
const result = event.results[i];
|
||||
if (result.isFinal) {
|
||||
finalTranscript += result[0].transcript;
|
||||
} else {
|
||||
interimTranscript += result[0].transcript;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[BrowserVoiceService] Transcripts - interim:', interimTranscript, 'final:', finalTranscript);
|
||||
|
||||
// Send interim results
|
||||
if (interimTranscript) {
|
||||
console.log('[BrowserVoiceService] Calling onResultCallback with interim');
|
||||
this.onResultCallback?.(interimTranscript, false);
|
||||
}
|
||||
|
||||
// Send final results
|
||||
if (finalTranscript) {
|
||||
console.log('[BrowserVoiceService] Calling onResultCallback with final');
|
||||
this.onResultCallback?.(finalTranscript, true);
|
||||
}
|
||||
};
|
||||
|
||||
this.recognition.onerror = (event: SpeechRecognitionErrorEvent) => {
|
||||
// "aborted" is commonly emitted when we intentionally stop/pause recognition.
|
||||
// Treat it as non-fatal to avoid noisy error loops in continuous mode.
|
||||
if (event.error === 'aborted') {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorMessage = this.getErrorMessage(event.error);
|
||||
this.onErrorCallback?.(errorMessage);
|
||||
|
||||
// Don't restart on certain errors
|
||||
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') {
|
||||
this.restartOnEnd = false;
|
||||
this.isListening = false;
|
||||
}
|
||||
};
|
||||
|
||||
this.recognition.onend = () => {
|
||||
this.isListening = false;
|
||||
|
||||
// Auto-restart if still supposed to be listening and not speaking
|
||||
if (this.restartOnEnd && this.recognition && !this.isSpeaking) {
|
||||
try {
|
||||
this.recognition.start();
|
||||
} catch {
|
||||
// Ignore restart errors
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Start recognition - MUST be synchronous for iOS Safari
|
||||
try {
|
||||
this.recognition.start();
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to start speech recognition';
|
||||
onError?.(errorMsg);
|
||||
throw new Error(errorMsg);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start speech recognition (async version for desktop/backward compatibility)
|
||||
* @param lang - BCP 47 language tag (e.g., 'en-US', 'es-ES')
|
||||
* @param onResult - Callback for speech results
|
||||
* @param onError - Optional callback for errors
|
||||
* @returns Promise that resolves when recognition starts
|
||||
*/
|
||||
async startListening(
|
||||
lang: string,
|
||||
onResult: SpeechResultCallback,
|
||||
onError?: ErrorCallback
|
||||
): Promise<void> {
|
||||
// Start recognition directly from the user gesture path.
|
||||
// Some webview runtimes reject preflight getUserMedia and then never show
|
||||
// permission prompt, while SpeechRecognition.start() can still trigger it.
|
||||
this.startListeningSync(lang, onResult, onError);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop speech recognition
|
||||
*/
|
||||
stopListening(): void {
|
||||
this.restartOnEnd = false;
|
||||
|
||||
if (this.recognition) {
|
||||
try {
|
||||
this.recognition.stop();
|
||||
} catch {
|
||||
// Ignore stop errors
|
||||
}
|
||||
this.recognition = null;
|
||||
}
|
||||
|
||||
this.isListening = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if currently listening
|
||||
*/
|
||||
getIsListening(): boolean {
|
||||
return this.isListening;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current language
|
||||
*/
|
||||
getCurrentLang(): string {
|
||||
return this.currentLang;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume audio context to unlock audio for playback
|
||||
* Must be called within a user gesture handler
|
||||
*/
|
||||
async resumeAudioContext(): Promise<void> {
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if audio unlock is required (autoplay policy blocked audio)
|
||||
*/
|
||||
isAudioUnlockRequired(): boolean {
|
||||
return this.audioUnlockRequired;
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually unlock audio by playing a silent sound
|
||||
* Call this from a button click handler for stubborn browsers
|
||||
*/
|
||||
async unlockAudio(): Promise<boolean> {
|
||||
try {
|
||||
// Create and play silent audio to unlock Web Audio API
|
||||
const audio = new Audio();
|
||||
// 1ms silence WAV file (base64 encoded)
|
||||
audio.src = 'data:audio/wav;base64,UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA';
|
||||
audio.volume = 0.01;
|
||||
await audio.play();
|
||||
|
||||
// Resume audio context
|
||||
await this.resumeAudioContext();
|
||||
|
||||
// Also unlock speech synthesis on mobile Safari by speaking a silent utterance
|
||||
// This must be done within a user gesture to allow future speech
|
||||
if (this.isMobileDevice() && 'speechSynthesis' in window) {
|
||||
const unlockUtterance = new SpeechSynthesisUtterance('');
|
||||
unlockUtterance.volume = 0;
|
||||
window.speechSynthesis.speak(unlockUtterance);
|
||||
window.speechSynthesis.cancel(); // Cancel immediately
|
||||
console.log('[BrowserVoiceService] Speech synthesis unlocked for mobile');
|
||||
}
|
||||
|
||||
this.audioUnlockRequired = false;
|
||||
console.log('[BrowserVoiceService] Audio unlocked successfully');
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('[BrowserVoiceService] Failed to unlock audio:', err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Speak text using speech synthesis
|
||||
* @param text - Text to speak
|
||||
* @param lang - BCP 47 language tag for voice selection
|
||||
* @param onEnd - Optional callback when speech ends
|
||||
* @param options - Optional TTS configuration (rate, pitch, volume, voiceName)
|
||||
* @returns Promise that resolves when speech starts
|
||||
*/
|
||||
async speakText(
|
||||
text: string,
|
||||
lang: string,
|
||||
onEnd?: SpeechEndCallback,
|
||||
options?: { rate?: number; pitch?: number; volume?: number; voiceName?: string }
|
||||
): Promise<void> {
|
||||
if (!('speechSynthesis' in window)) {
|
||||
throw new Error('Speech synthesis not supported');
|
||||
}
|
||||
|
||||
// Resume audio context first (user gesture must have happened)
|
||||
await this.resumeAudioContext();
|
||||
|
||||
// Wait for voices to be available (Chrome requires this)
|
||||
const voices = await this.waitForVoices();
|
||||
|
||||
// Small delay to ensure audio context is ready
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
|
||||
// Set speaking state and pause listening to avoid hearing ourselves
|
||||
this.isSpeaking = true;
|
||||
this.pauseListening();
|
||||
|
||||
// Cancel any ongoing speech
|
||||
window.speechSynthesis.cancel();
|
||||
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.lang = lang;
|
||||
utterance.rate = options?.rate ?? 1;
|
||||
utterance.pitch = options?.pitch ?? 1;
|
||||
utterance.volume = options?.volume ?? 1;
|
||||
|
||||
// Try to find voice by name first (user-selected), then fallback to language match
|
||||
let selectedVoice: SpeechSynthesisVoice | null = null;
|
||||
|
||||
if (options?.voiceName) {
|
||||
selectedVoice = voices.find(v => v.name === options.voiceName) || null;
|
||||
if (selectedVoice) {
|
||||
console.log(`[BrowserVoiceService] Using selected voice: ${selectedVoice.name} (${selectedVoice.lang})`);
|
||||
} else {
|
||||
console.warn(`[BrowserVoiceService] Selected voice "${options.voiceName}" not found, falling back to language match`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!selectedVoice) {
|
||||
selectedVoice = this.findBestVoice(voices, lang);
|
||||
if (selectedVoice) {
|
||||
console.log(`[BrowserVoiceService] Using language-matched voice: ${selectedVoice.name} (${selectedVoice.lang})`);
|
||||
} else {
|
||||
console.warn(`[BrowserVoiceService] No voice found for language: ${lang}, using default`);
|
||||
}
|
||||
}
|
||||
|
||||
if (selectedVoice) {
|
||||
utterance.voice = selectedVoice;
|
||||
}
|
||||
|
||||
console.log(`[BrowserVoiceService] Speaking text (${text.length} chars) in ${lang}`);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let hasStarted = false;
|
||||
|
||||
utterance.onstart = () => {
|
||||
hasStarted = true;
|
||||
console.log('[BrowserVoiceService] Speech started');
|
||||
resolve();
|
||||
};
|
||||
|
||||
utterance.onend = () => {
|
||||
this.isSpeaking = false;
|
||||
console.log('[BrowserVoiceService] Speech ended');
|
||||
onEnd?.();
|
||||
};
|
||||
|
||||
utterance.onerror = (event) => {
|
||||
this.isSpeaking = false;
|
||||
console.error('[BrowserVoiceService] Speech synthesis error:', event.error);
|
||||
|
||||
// Track autoplay policy violations
|
||||
if (event.error === 'not-allowed' || event.error === 'interrupted') {
|
||||
this.audioUnlockRequired = true;
|
||||
}
|
||||
|
||||
// Provide more specific error messages
|
||||
let errorMessage = `Speech synthesis error: ${event.error || 'unknown'}`;
|
||||
if (event.error === 'not-allowed') {
|
||||
errorMessage = 'Audio blocked by browser autoplay policy. Please interact with the page first.';
|
||||
} else if (event.error === 'interrupted') {
|
||||
errorMessage = 'Speech was interrupted. Please try again.';
|
||||
}
|
||||
|
||||
reject(new Error(errorMessage));
|
||||
};
|
||||
|
||||
// Safety timeout - if onstart doesn't fire within 2 seconds, something is wrong
|
||||
setTimeout(() => {
|
||||
if (!hasStarted) {
|
||||
console.warn('[BrowserVoiceService] Speech start timeout - audio may be blocked');
|
||||
this.audioUnlockRequired = true;
|
||||
}
|
||||
}, 2000);
|
||||
|
||||
window.speechSynthesis.speak(utterance);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel ongoing speech
|
||||
*/
|
||||
cancelSpeech(): void {
|
||||
if ('speechSynthesis' in window) {
|
||||
window.speechSynthesis.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available voices
|
||||
*/
|
||||
getVoices(): SpeechSynthesisVoice[] {
|
||||
if (!('speechSynthesis' in window)) {
|
||||
return [];
|
||||
}
|
||||
return window.speechSynthesis.getVoices();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for voices to load (needed for Chrome)
|
||||
*/
|
||||
async waitForVoices(): Promise<SpeechSynthesisVoice[]> {
|
||||
if (!('speechSynthesis' in window)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const voices = window.speechSynthesis.getVoices();
|
||||
if (voices.length > 0) {
|
||||
return voices;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const handleVoicesChanged = () => {
|
||||
resolve(window.speechSynthesis.getVoices());
|
||||
window.speechSynthesis.onvoiceschanged = null;
|
||||
};
|
||||
|
||||
window.speechSynthesis.onvoiceschanged = handleVoicesChanged;
|
||||
|
||||
// Timeout fallback
|
||||
setTimeout(() => {
|
||||
resolve(window.speechSynthesis.getVoices());
|
||||
window.speechSynthesis.onvoiceschanged = null;
|
||||
}, 1000);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the best voice for a given language
|
||||
*/
|
||||
private findBestVoice(voices: SpeechSynthesisVoice[], lang: string): SpeechSynthesisVoice | null {
|
||||
// First try exact match
|
||||
let voice = voices.find(v => v.lang === lang);
|
||||
|
||||
if (!voice) {
|
||||
// Try language code match (e.g., 'en' for 'en-US')
|
||||
const langCode = lang.split('-')[0];
|
||||
voice = voices.find(v => v.lang.startsWith(langCode));
|
||||
}
|
||||
|
||||
if (!voice) {
|
||||
// Prefer local voices
|
||||
voice = voices.find(v => v.lang.startsWith(lang.split('-')[0]) && v.localService);
|
||||
}
|
||||
|
||||
return voice || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if running on mobile device
|
||||
*/
|
||||
private isMobileDevice(): boolean {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
return /iphone|ipad|ipod|android|mobile|webos|blackberry|iemobile|opera mini/i.test(userAgent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if running on iOS Safari
|
||||
*/
|
||||
private isIOSSafari(): boolean {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const userAgent = navigator.userAgent.toLowerCase();
|
||||
const isIOS = /iphone|ipad|ipod/i.test(userAgent);
|
||||
const isSafari = /safari/i.test(userAgent) && !/chrome|crios|crmo/i.test(userAgent);
|
||||
return isIOS && isSafari;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable error message
|
||||
* @param error - Error code from SpeechRecognition
|
||||
*/
|
||||
private getErrorMessage(error: string): string {
|
||||
const isMobileDevice = this.isMobileDevice();
|
||||
const isIOS = this.isIOSSafari();
|
||||
|
||||
const errorMessages: Record<string, string> = {
|
||||
'no-speech': 'No speech detected',
|
||||
'aborted': 'Speech recognition aborted',
|
||||
'audio-capture': 'No microphone found',
|
||||
'network': 'Network error - check connection',
|
||||
'not-allowed': isMobileDevice
|
||||
? 'Microphone permission denied. Check Settings > Safari > Microphone'
|
||||
: 'Microphone permission denied',
|
||||
'service-not-allowed': isIOS
|
||||
? 'Speech recognition requires a user gesture. Please tap the microphone button again.'
|
||||
: 'Speech recognition service not allowed',
|
||||
'bad-grammar': 'Grammar error',
|
||||
'language-not-supported': 'Language not supported',
|
||||
};
|
||||
|
||||
return errorMessages[error] || `Speech recognition error: ${error}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const browserVoiceService = new BrowserVoiceService();
|
||||
|
||||
// Also export the class for testing/customization
|
||||
export { BrowserVoiceService };
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Context formatters for voice-native output
|
||||
* Formats session events (messages, permissions, ready events) into natural language
|
||||
* for the ElevenLabs voice agent to speak aloud.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { formatMessage, formatPermissionRequest } from '@/lib/voice';
|
||||
*
|
||||
* const voiceText = formatMessage({ role: 'assistant', content: 'Hello!' });
|
||||
* // Returns: "Claude Code: Hello!"
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { VOICE_CONFIG } from "./voiceConfig";
|
||||
|
||||
/** Message type for voice formatting */
|
||||
export interface VoiceMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a single message for voice output
|
||||
* - Assistant messages: Code blocks replaced with "[code block]", prefixed with "Claude Code: "
|
||||
* - User messages: Prefixed with "User: "
|
||||
* - Other roles: Returns null (not spoken)
|
||||
*
|
||||
* @param message - The message to format
|
||||
* @returns Formatted text for voice, or null if should not be spoken
|
||||
*/
|
||||
export function formatMessage(message: VoiceMessage): string | null {
|
||||
// Handle edge cases
|
||||
if (!message || typeof message.content !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = message.content.trim();
|
||||
if (!content) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (message.role === "assistant") {
|
||||
// Replace code blocks with description (don't read code aloud)
|
||||
const textOnly = content.replace(/```[\s\S]*?```/g, "[code block]");
|
||||
return `Claude Code: ${textOnly}`;
|
||||
}
|
||||
|
||||
if (message.role === "user") {
|
||||
return `User: ${content}`;
|
||||
}
|
||||
|
||||
// Skip system, tool, and other roles for voice
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format multiple new messages for voice output
|
||||
* - Maps messages through formatMessage
|
||||
* - Filters out nulls (unspoken roles)
|
||||
* - Joins with newlines
|
||||
*
|
||||
* @param sessionId - The session ID (for future use/debugging)
|
||||
* @param messages - Array of messages to format
|
||||
* @returns Formatted text for voice, or null if no speakable messages
|
||||
*/
|
||||
export function formatNewMessages(
|
||||
sessionId: string,
|
||||
messages: VoiceMessage[]
|
||||
): string | null {
|
||||
// Handle edge cases
|
||||
if (!Array.isArray(messages) || messages.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log(`[Voice] Formatting ${messages.length} messages for session ${sessionId}`);
|
||||
}
|
||||
|
||||
// Format each message and filter out nulls
|
||||
const formattedMessages = messages
|
||||
.map(formatMessage)
|
||||
.filter((msg): msg is string => msg !== null);
|
||||
|
||||
if (formattedMessages.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return formattedMessages.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a permission request for voice announcement
|
||||
* - Per CONTEXT.md: Only tool name, not arguments (LIMITED_TOOL_CALLS)
|
||||
* - Prompts user to say "allow" or "deny"
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @param requestId - The permission request ID
|
||||
* @param toolName - Name of the tool requesting permission
|
||||
* @param toolArgs - Tool arguments (not included in voice output per config)
|
||||
* @returns Formatted permission request for voice
|
||||
*/
|
||||
export function formatPermissionRequest(
|
||||
sessionId: string,
|
||||
requestId: string,
|
||||
toolName: string,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
toolArgs: unknown
|
||||
): string {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log(`[Voice] Formatting permission request ${requestId} for session ${sessionId}`);
|
||||
}
|
||||
|
||||
// Per VOICE_CONFIG.LIMITED_TOOL_CALLS, we don't include toolArgs in voice output
|
||||
return `Claude Code is requesting permission to use ${toolName}. Say "allow" or "deny".`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a ready event for voice announcement
|
||||
* - Indicates the AI has finished working and is ready for next instruction
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @returns Formatted ready event for voice
|
||||
*/
|
||||
export function formatReadyEvent(sessionId: string): string {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log(`[Voice] Formatting ready event for session ${sessionId}`);
|
||||
}
|
||||
|
||||
return "Claude Code finished working. Ready for next instruction.";
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Voice module barrel export
|
||||
* Provides clean import path for voice configuration and client tools
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { VOICE_CONFIG, realtimeClientTools, voiceHooks } from '@/lib/voice';
|
||||
* ```
|
||||
*/
|
||||
|
||||
// Configuration
|
||||
export { VOICE_CONFIG } from "./voiceConfig";
|
||||
|
||||
// Client tools for ElevenLabs voice agent
|
||||
export { realtimeClientTools } from "./realtimeClientTools";
|
||||
export type { RealtimeClientTools } from "./realtimeClientTools";
|
||||
|
||||
// Voice session registry (from voiceSession.ts)
|
||||
export {
|
||||
registerVoiceSession,
|
||||
unregisterVoiceSession,
|
||||
getVoiceSession,
|
||||
isVoiceSessionStarted,
|
||||
} from "./voiceSession";
|
||||
|
||||
// Voice hooks for session-to-voice event routing (from voiceHooks.ts)
|
||||
export { voiceHooks } from "./voiceHooks";
|
||||
|
||||
// Context formatters for voice-native output
|
||||
export {
|
||||
formatMessage,
|
||||
formatNewMessages,
|
||||
formatPermissionRequest,
|
||||
formatReadyEvent,
|
||||
type VoiceMessage,
|
||||
} from "./contextFormatters";
|
||||
@@ -0,0 +1,106 @@
|
||||
import { z } from "zod";
|
||||
import { useSessionStore } from "@/stores/useSessionStore";
|
||||
import { useConfigStore } from "@/stores/useConfigStore";
|
||||
import { usePermissionStore } from "@/stores/permissionStore";
|
||||
|
||||
/**
|
||||
* Static client tools for the realtime voice interface.
|
||||
* These tools allow the voice agent to interact with Claude Code.
|
||||
*/
|
||||
export const realtimeClientTools = {
|
||||
/**
|
||||
* Send a message to Claude Code via the current session.
|
||||
* Validates parameters with Zod and returns status strings.
|
||||
*/
|
||||
messageClaudeCode: async (parameters: unknown): Promise<string> => {
|
||||
// Validate parameters with Zod
|
||||
const schema = z.object({
|
||||
message: z.string().min(1, "Message cannot be empty"),
|
||||
});
|
||||
|
||||
const parsed = schema.safeParse(parameters);
|
||||
if (!parsed.success) {
|
||||
console.error("[Voice] Invalid message parameter:", parsed.error);
|
||||
return "error (invalid message parameter)";
|
||||
}
|
||||
|
||||
// Get current session ID from store
|
||||
const sessionId = useSessionStore.getState().currentSessionId;
|
||||
if (!sessionId) {
|
||||
console.error("[Voice] No active session");
|
||||
return "error (no active session)";
|
||||
}
|
||||
|
||||
// Get current provider and model from config store
|
||||
const { currentProviderId, currentModelId, currentAgentName } = useConfigStore.getState();
|
||||
if (!currentProviderId || !currentModelId) {
|
||||
console.error("[Voice] No provider/model selected");
|
||||
return "error (no provider or model selected)";
|
||||
}
|
||||
|
||||
try {
|
||||
console.log("[Voice] Sending message to session:", sessionId);
|
||||
await useSessionStore
|
||||
.getState()
|
||||
.sendMessage(parsed.data.message, currentProviderId, currentModelId, currentAgentName ?? undefined);
|
||||
return "sent";
|
||||
} catch (error) {
|
||||
console.error("[Voice] Failed to send message:", error);
|
||||
return "error (failed to send message)";
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Process a permission request from voice.
|
||||
* Validates decision with Zod enum and interacts with permission store.
|
||||
*/
|
||||
processPermissionRequest: async (parameters: unknown): Promise<string> => {
|
||||
// Validate parameters with Zod
|
||||
const schema = z.object({
|
||||
decision: z.enum(["allow", "deny"]),
|
||||
});
|
||||
|
||||
const parsed = schema.safeParse(parameters);
|
||||
if (!parsed.success) {
|
||||
console.error("[Voice] Invalid decision parameter:", parsed.error);
|
||||
return "error (invalid decision parameter, expected 'allow' or 'deny')";
|
||||
}
|
||||
|
||||
// Get current session ID from store
|
||||
const sessionId = useSessionStore.getState().currentSessionId;
|
||||
if (!sessionId) {
|
||||
console.error("[Voice] No active session");
|
||||
return "error (no active session)";
|
||||
}
|
||||
|
||||
// Get pending permissions for this session
|
||||
const permissions = usePermissionStore.getState().permissions.get(sessionId);
|
||||
if (!permissions || permissions.length === 0) {
|
||||
console.error("[Voice] No pending permission requests");
|
||||
return "error (no pending permission request)";
|
||||
}
|
||||
|
||||
// Get the first pending permission request
|
||||
const request = permissions[0];
|
||||
if (!request) {
|
||||
return "error (no pending permission request)";
|
||||
}
|
||||
|
||||
try {
|
||||
const decision = parsed.data.decision;
|
||||
console.log(`[Voice] Processing permission request ${request.id}: ${decision}`);
|
||||
|
||||
// Respond to the permission based on decision
|
||||
const response: "once" | "always" | "reject" = decision === "allow" ? "once" : "reject";
|
||||
await usePermissionStore.getState().respondToPermission(sessionId, request.id, response);
|
||||
|
||||
return "done";
|
||||
} catch (error) {
|
||||
console.error("[Voice] Failed to process permission:", error);
|
||||
return `error (failed to ${parsed.data.decision} permission)`;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
/** Type for the realtime client tools */
|
||||
export type RealtimeClientTools = typeof realtimeClientTools;
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Text summarization utility for TTS
|
||||
*
|
||||
* Calls the server-side summarization endpoint which uses
|
||||
* the opencode.ai zen API with gpt-5-nano.
|
||||
*/
|
||||
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
/**
|
||||
* Summarize text using the server-side zen API endpoint
|
||||
*
|
||||
* @param text - The text to summarize
|
||||
* @param options - Optional configuration
|
||||
* @returns The summarized text, or original text if summarization fails
|
||||
*/
|
||||
export async function summarizeText(
|
||||
text: string,
|
||||
options?: {
|
||||
/** Character threshold - don't summarize if under this length */
|
||||
threshold?: number;
|
||||
/** Max characters for the summary output */
|
||||
maxLength?: number;
|
||||
}
|
||||
): Promise<string> {
|
||||
const store = useConfigStore.getState();
|
||||
const threshold = options?.threshold ?? store.summarizeCharacterThreshold;
|
||||
const maxLength = options?.maxLength ?? store.summarizeMaxLength;
|
||||
|
||||
// Don't summarize if text is under threshold
|
||||
if (text.length <= threshold) {
|
||||
return text;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/tts/summarize', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ text, threshold, maxLength }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error(`[summarize] HTTP error ${response.status}:`, errorText);
|
||||
throw new Error(`Summarization failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as {
|
||||
summarized: boolean;
|
||||
summary?: string;
|
||||
reason?: string;
|
||||
originalLength?: number;
|
||||
summaryLength?: number;
|
||||
};
|
||||
|
||||
if (data.summarized && data.summary) {
|
||||
return data.summary;
|
||||
}
|
||||
|
||||
// Return original text if not summarized
|
||||
return text;
|
||||
} catch (err) {
|
||||
console.error('[summarize] Failed to summarize:', err);
|
||||
// Return original text on error
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text should be summarized based on settings
|
||||
*/
|
||||
export function shouldSummarize(
|
||||
text: string,
|
||||
context: 'message' | 'voice'
|
||||
): boolean {
|
||||
const store = useConfigStore.getState();
|
||||
|
||||
const isEnabled = context === 'message'
|
||||
? store.summarizeMessageTTS
|
||||
: store.summarizeVoiceConversation;
|
||||
|
||||
if (!isEnabled) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return text.length > store.summarizeCharacterThreshold;
|
||||
}
|
||||
|
||||
/**
|
||||
* Client-side text sanitization for TTS output.
|
||||
* Removes markdown, URLs, file paths, and other non-speakable content.
|
||||
* Applied as a fallback when server-side summarization is skipped.
|
||||
*/
|
||||
export function sanitizeForTTS(text: string): string {
|
||||
if (!text) return '';
|
||||
return text
|
||||
// Remove code blocks
|
||||
.replace(/```[\s\S]*?```/g, '')
|
||||
.replace(/`[^`]*`/g, '')
|
||||
// Remove markdown formatting
|
||||
.replace(/[*_~#]/g, '')
|
||||
// Remove URLs
|
||||
.replace(/https?:\/\/[^\s]+/g, '')
|
||||
// Remove file paths
|
||||
.replace(/\/[\w\-./]+/g, '')
|
||||
// Remove shell-like patterns
|
||||
.replace(/^\s*[$#>]\s*/gm, '')
|
||||
// Remove brackets and special chars
|
||||
.replace(/[[\]{}()<>|&;]/g, ' ')
|
||||
.replace(/\\/g, '')
|
||||
// Collapse whitespace
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Static voice context configuration
|
||||
* Controls voice behavior and feature flags for the ElevenLabs voice agent
|
||||
*/
|
||||
export const VOICE_CONFIG = {
|
||||
/** Disable all tool call information from being sent to voice context */
|
||||
DISABLE_TOOL_CALLS: false,
|
||||
|
||||
/** Send only tool names and descriptions, exclude arguments */
|
||||
LIMITED_TOOL_CALLS: true,
|
||||
|
||||
/** Disable permission request forwarding */
|
||||
DISABLE_PERMISSION_REQUESTS: false,
|
||||
|
||||
/** Disable session online/offline notifications */
|
||||
DISABLE_SESSION_STATUS: true,
|
||||
|
||||
/** Disable message forwarding */
|
||||
DISABLE_MESSAGES: false,
|
||||
|
||||
/** Disable session focus notifications */
|
||||
DISABLE_SESSION_FOCUS: false,
|
||||
|
||||
/** Disable ready event notifications */
|
||||
DISABLE_READY_EVENTS: false,
|
||||
|
||||
/** Maximum number of messages to include in session history */
|
||||
MAX_HISTORY_MESSAGES: 50,
|
||||
|
||||
/** Enable debug logging for voice context updates */
|
||||
ENABLE_DEBUG_LOGGING: true,
|
||||
} as const;
|
||||
|
||||
/** Type for VOICE_CONFIG keys */
|
||||
export type VoiceConfigKey = keyof typeof VOICE_CONFIG;
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Voice hooks for session-to-voice event routing
|
||||
* Routes session events (messages, permissions, ready events) to the ElevenLabs
|
||||
* voice agent via contextual updates.
|
||||
*
|
||||
* This module provides hooks that can be called when session events occur,
|
||||
* using the voice session registry from voiceSession.ts.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { voiceHooks } from '@/lib/voice';
|
||||
*
|
||||
* // Route session messages to voice
|
||||
* voiceHooks.onMessages(sessionId, messages);
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { VOICE_CONFIG } from "./voiceConfig";
|
||||
import {
|
||||
formatNewMessages,
|
||||
formatPermissionRequest,
|
||||
formatReadyEvent,
|
||||
type VoiceMessage,
|
||||
} from "./contextFormatters";
|
||||
import { getVoiceSession, isVoiceSessionStarted } from "./voiceSession";
|
||||
|
||||
// Re-export registry functions from voiceSession.ts for convenience
|
||||
export {
|
||||
registerVoiceSession,
|
||||
unregisterVoiceSession,
|
||||
getVoiceSession,
|
||||
isVoiceSessionStarted,
|
||||
} from "./voiceSession";
|
||||
|
||||
/**
|
||||
* Report a contextual update to the voice session
|
||||
* Internal helper that checks preconditions and handles errors
|
||||
*
|
||||
* @param update - The text update to send, or null/undefined to skip
|
||||
*/
|
||||
function reportContextualUpdate(update: string | null | undefined): void {
|
||||
// Skip empty/null/undefined updates
|
||||
if (!update || update.trim().length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip if no voice session or not started
|
||||
const voiceSession = getVoiceSession();
|
||||
if (!voiceSession || !isVoiceSessionStarted()) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Skipping contextual update - no active session");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Sending contextual update:", update.substring(0, 100));
|
||||
}
|
||||
voiceSession.sendContextualUpdate(update);
|
||||
} catch (error) {
|
||||
// Log error but don't throw - voice updates shouldn't break the app
|
||||
console.error("[Voice] Failed to send contextual update:", error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Voice hooks - exported functions to route session events to voice
|
||||
*
|
||||
* These hooks should be called when corresponding session events occur.
|
||||
* They respect VOICE_CONFIG feature flags to enable/disable specific
|
||||
* event types.
|
||||
*/
|
||||
export const voiceHooks = {
|
||||
/**
|
||||
* Called when new messages arrive in the session
|
||||
* Formats and sends messages to voice agent (if not disabled)
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @param messages - Array of messages to format and send
|
||||
*/
|
||||
onMessages(sessionId: string, messages: VoiceMessage[]): void {
|
||||
if (VOICE_CONFIG.DISABLE_MESSAGES) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Message forwarding disabled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
reportContextualUpdate(formatNewMessages(sessionId, messages));
|
||||
},
|
||||
|
||||
/**
|
||||
* Called when a permission request is made
|
||||
* Announces the permission request to the voice agent (if not disabled)
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
* @param requestId - The permission request ID
|
||||
* @param toolName - Name of the tool requesting permission
|
||||
* @param toolArgs - Arguments for the tool (not sent to voice per LIMITED_TOOL_CALLS)
|
||||
*/
|
||||
onPermissionRequested(
|
||||
sessionId: string,
|
||||
requestId: string,
|
||||
toolName: string,
|
||||
toolArgs: unknown
|
||||
): void {
|
||||
if (VOICE_CONFIG.DISABLE_PERMISSION_REQUESTS) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Permission request forwarding disabled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
reportContextualUpdate(
|
||||
formatPermissionRequest(sessionId, requestId, toolName, toolArgs)
|
||||
);
|
||||
},
|
||||
|
||||
/**
|
||||
* Called when the AI is ready for the next instruction
|
||||
* Announces ready state to the voice agent (if not disabled)
|
||||
*
|
||||
* @param sessionId - The session ID
|
||||
*/
|
||||
onReady(sessionId: string): void {
|
||||
if (VOICE_CONFIG.DISABLE_READY_EVENTS) {
|
||||
if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) {
|
||||
console.log("[Voice] Ready event forwarding disabled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
reportContextualUpdate(formatReadyEvent(sessionId));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Voice session interface
|
||||
* Used for type safety without importing ReturnType from SDK
|
||||
*/
|
||||
interface VoiceSession {
|
||||
sendContextualUpdate: (text: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Global storage for the active voice session.
|
||||
* Used by voiceHooks to send contextual updates to the voice agent.
|
||||
*/
|
||||
let activeVoiceSession: VoiceSession | null = null;
|
||||
|
||||
/**
|
||||
* Register a voice session for use by voiceHooks.
|
||||
* Called by useVoice when a conversation is established.
|
||||
*/
|
||||
export function registerVoiceSession(session: VoiceSession): void {
|
||||
activeVoiceSession = session;
|
||||
console.log("[Voice] Session registered");
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister the active voice session.
|
||||
* Called by useVoice when the session ends.
|
||||
*/
|
||||
export function unregisterVoiceSession(): void {
|
||||
activeVoiceSession = null;
|
||||
console.log("[Voice] Session unregistered");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the currently registered voice session.
|
||||
* Used by voiceHooks to send contextual updates.
|
||||
*/
|
||||
export function getVoiceSession(): VoiceSession | null {
|
||||
return activeVoiceSession;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a voice session is currently active.
|
||||
*/
|
||||
export function isVoiceSessionStarted(): boolean {
|
||||
return activeVoiceSession !== null;
|
||||
}
|
||||
@@ -105,6 +105,15 @@ export type NewSessionDraftState = {
|
||||
syntheticParts?: SyntheticContextPart[];
|
||||
};
|
||||
|
||||
// Voice state types
|
||||
export type VoiceStatus = 'disconnected' | 'connecting' | 'connected' | 'error';
|
||||
export type VoiceMode = 'idle' | 'speaking' | 'listening';
|
||||
|
||||
export interface VoiceState {
|
||||
status: VoiceStatus;
|
||||
mode: VoiceMode;
|
||||
}
|
||||
|
||||
export interface SessionStore {
|
||||
|
||||
sessions: Session[];
|
||||
@@ -171,6 +180,14 @@ export interface SessionStore {
|
||||
|
||||
newSessionDraft: NewSessionDraftState;
|
||||
|
||||
// Voice state
|
||||
voiceStatus: VoiceStatus;
|
||||
voiceMode: VoiceMode;
|
||||
|
||||
// Voice actions
|
||||
setVoiceStatus: (status: VoiceStatus) => void;
|
||||
setVoiceMode: (mode: VoiceMode) => void;
|
||||
|
||||
getSessionAgentEditMode: (sessionId: string, agentName: string | undefined, defaultMode?: EditPermissionMode) => EditPermissionMode;
|
||||
toggleSessionAgentEditMode: (sessionId: string, agentName: string | undefined, defaultMode?: EditPermissionMode) => void;
|
||||
setSessionAgentEditMode: (sessionId: string, agentName: string | undefined, mode: EditPermissionMode, defaultMode?: EditPermissionMode) => void;
|
||||
|
||||
@@ -366,6 +366,37 @@ interface ConfigStore {
|
||||
settingsDefaultAgent: string | undefined;
|
||||
settingsAutoCreateWorktree: boolean;
|
||||
settingsGitmojiEnabled: boolean;
|
||||
// Voice provider preference ('browser', 'openai', or 'say' for macOS)
|
||||
voiceProvider: 'browser' | 'openai' | 'say';
|
||||
setVoiceProvider: (provider: 'browser' | 'openai' | 'say') => void;
|
||||
// TTS settings
|
||||
speechRate: number;
|
||||
speechPitch: number;
|
||||
speechVolume: number;
|
||||
sayVoice: string;
|
||||
browserVoice: string;
|
||||
openaiVoice: string;
|
||||
openaiApiKey: string;
|
||||
showMessageTTSButtons: boolean;
|
||||
voiceModeEnabled: boolean;
|
||||
// Summarization settings
|
||||
summarizeMessageTTS: boolean;
|
||||
summarizeVoiceConversation: boolean;
|
||||
summarizeCharacterThreshold: number;
|
||||
summarizeMaxLength: number;
|
||||
setSpeechRate: (rate: number) => void;
|
||||
setSpeechPitch: (pitch: number) => void;
|
||||
setSpeechVolume: (volume: number) => void;
|
||||
setSayVoice: (voice: string) => void;
|
||||
setBrowserVoice: (voice: string) => void;
|
||||
setOpenaiVoice: (voice: string) => void;
|
||||
setOpenaiApiKey: (apiKey: string) => void;
|
||||
setShowMessageTTSButtons: (show: boolean) => void;
|
||||
setVoiceModeEnabled: (enabled: boolean) => void;
|
||||
setSummarizeMessageTTS: (enabled: boolean) => void;
|
||||
setSummarizeVoiceConversation: (enabled: boolean) => void;
|
||||
setSummarizeCharacterThreshold: (threshold: number) => void;
|
||||
setSummarizeMaxLength: (maxLength: number) => void;
|
||||
|
||||
activateDirectory: (directory: string | null | undefined) => Promise<void>;
|
||||
|
||||
@@ -427,7 +458,128 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
settingsDefaultAgent: undefined,
|
||||
settingsAutoCreateWorktree: false,
|
||||
settingsGitmojiEnabled: false,
|
||||
|
||||
// Voice provider preference - load from localStorage or default to 'browser'
|
||||
voiceProvider: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('voiceProvider');
|
||||
if (saved === 'openai' || saved === 'browser' || saved === 'say') return saved;
|
||||
}
|
||||
return 'browser';
|
||||
})(),
|
||||
// TTS settings - load from localStorage with defaults
|
||||
speechRate: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('speechRate');
|
||||
if (saved) {
|
||||
const parsed = parseFloat(saved);
|
||||
if (!isNaN(parsed) && parsed >= 0.5 && parsed <= 2) return parsed;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
})(),
|
||||
speechPitch: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('speechPitch');
|
||||
if (saved) {
|
||||
const parsed = parseFloat(saved);
|
||||
if (!isNaN(parsed) && parsed >= 0.5 && parsed <= 2) return parsed;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
})(),
|
||||
speechVolume: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('speechVolume');
|
||||
if (saved) {
|
||||
const parsed = parseFloat(saved);
|
||||
if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) return parsed;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
})(),
|
||||
// macOS Say voice - load from localStorage or default to 'Samantha'
|
||||
sayVoice: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('sayVoice');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return 'Samantha';
|
||||
})(),
|
||||
// Browser voice - load from localStorage or default to empty (auto-select)
|
||||
browserVoice: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('browserVoice');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
// OpenAI voice - load from localStorage or default to 'nova'
|
||||
openaiVoice: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('openaiVoice');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return 'nova';
|
||||
})(),
|
||||
// OpenAI API key for TTS - load from localStorage or default to empty
|
||||
openaiApiKey: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('openaiApiKey');
|
||||
if (saved) return saved;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
// Show TTS buttons on messages - load from localStorage or default to true
|
||||
showMessageTTSButtons: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('showMessageTTSButtons');
|
||||
if (saved === 'false') return false;
|
||||
}
|
||||
return true;
|
||||
})(),
|
||||
// Voice mode enabled - load from localStorage or default to true
|
||||
voiceModeEnabled: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('voiceModeEnabled');
|
||||
if (saved === 'false') return false;
|
||||
}
|
||||
return true;
|
||||
})(),
|
||||
// Summarization settings
|
||||
summarizeMessageTTS: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('summarizeMessageTTS');
|
||||
if (saved === 'true') return true;
|
||||
}
|
||||
return false;
|
||||
})(),
|
||||
summarizeVoiceConversation: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('summarizeVoiceConversation');
|
||||
if (saved === 'true') return true;
|
||||
}
|
||||
return false;
|
||||
})(),
|
||||
summarizeCharacterThreshold: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('summarizeCharacterThreshold');
|
||||
if (saved) {
|
||||
const parsed = parseInt(saved, 10);
|
||||
if (!isNaN(parsed) && parsed >= 50 && parsed <= 2000) return parsed;
|
||||
}
|
||||
}
|
||||
return 200;
|
||||
})(),
|
||||
summarizeMaxLength: (() => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const saved = localStorage.getItem('summarizeMaxLength');
|
||||
if (saved) {
|
||||
const parsed = parseInt(saved, 10);
|
||||
if (!isNaN(parsed) && parsed >= 50 && parsed <= 2000) return parsed;
|
||||
}
|
||||
}
|
||||
return 500;
|
||||
})(),
|
||||
activateDirectory: async (directory) => {
|
||||
const directoryKey = toDirectoryKey(directory);
|
||||
|
||||
@@ -1295,6 +1447,109 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
set({ settingsGitmojiEnabled: enabled });
|
||||
},
|
||||
|
||||
setVoiceProvider: (provider: 'browser' | 'openai' | 'say') => {
|
||||
set({ voiceProvider: provider });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('voiceProvider', provider);
|
||||
}
|
||||
},
|
||||
|
||||
setSpeechRate: (rate: number) => {
|
||||
const clampedRate = Math.max(0.5, Math.min(2, rate));
|
||||
set({ speechRate: clampedRate });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('speechRate', String(clampedRate));
|
||||
}
|
||||
},
|
||||
|
||||
setSpeechPitch: (pitch: number) => {
|
||||
const clampedPitch = Math.max(0.5, Math.min(2, pitch));
|
||||
set({ speechPitch: clampedPitch });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('speechPitch', String(clampedPitch));
|
||||
}
|
||||
},
|
||||
|
||||
setSpeechVolume: (volume: number) => {
|
||||
const clampedVolume = Math.max(0, Math.min(1, volume));
|
||||
set({ speechVolume: clampedVolume });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('speechVolume', String(clampedVolume));
|
||||
}
|
||||
},
|
||||
|
||||
setSayVoice: (voice: string) => {
|
||||
set({ sayVoice: voice });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('sayVoice', voice);
|
||||
}
|
||||
},
|
||||
|
||||
setBrowserVoice: (voice: string) => {
|
||||
set({ browserVoice: voice });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('browserVoice', voice);
|
||||
}
|
||||
},
|
||||
|
||||
setOpenaiVoice: (voice: string) => {
|
||||
set({ openaiVoice: voice });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('openaiVoice', voice);
|
||||
}
|
||||
},
|
||||
|
||||
setOpenaiApiKey: (apiKey: string) => {
|
||||
set({ openaiApiKey: apiKey });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('openaiApiKey', apiKey);
|
||||
}
|
||||
},
|
||||
|
||||
setShowMessageTTSButtons: (show: boolean) => {
|
||||
set({ showMessageTTSButtons: show });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('showMessageTTSButtons', String(show));
|
||||
}
|
||||
},
|
||||
|
||||
setVoiceModeEnabled: (enabled: boolean) => {
|
||||
set({ voiceModeEnabled: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('voiceModeEnabled', String(enabled));
|
||||
}
|
||||
},
|
||||
|
||||
setSummarizeMessageTTS: (enabled: boolean) => {
|
||||
set({ summarizeMessageTTS: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('summarizeMessageTTS', String(enabled));
|
||||
}
|
||||
},
|
||||
|
||||
setSummarizeVoiceConversation: (enabled: boolean) => {
|
||||
set({ summarizeVoiceConversation: enabled });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('summarizeVoiceConversation', String(enabled));
|
||||
}
|
||||
},
|
||||
|
||||
setSummarizeCharacterThreshold: (threshold: number) => {
|
||||
const clamped = Math.max(50, Math.min(2000, threshold));
|
||||
set({ summarizeCharacterThreshold: clamped });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('summarizeCharacterThreshold', String(clamped));
|
||||
}
|
||||
},
|
||||
|
||||
setSummarizeMaxLength: (maxLength: number) => {
|
||||
const clamped = Math.max(50, Math.min(2000, maxLength));
|
||||
set({ summarizeMaxLength: clamped });
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('summarizeMaxLength', String(clamped));
|
||||
}
|
||||
},
|
||||
|
||||
checkConnection: async () => {
|
||||
const maxAttempts = 5;
|
||||
let attempt = 0;
|
||||
@@ -1401,6 +1656,9 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
settingsDefaultAgent: state.settingsDefaultAgent,
|
||||
settingsAutoCreateWorktree: state.settingsAutoCreateWorktree,
|
||||
settingsGitmojiEnabled: state.settingsGitmojiEnabled,
|
||||
speechRate: state.speechRate,
|
||||
speechPitch: state.speechPitch,
|
||||
speechVolume: state.speechVolume,
|
||||
}),
|
||||
},
|
||||
),
|
||||
|
||||
@@ -106,6 +106,18 @@ export const useSessionStore = create<SessionStore>()(
|
||||
pendingSyntheticParts: null,
|
||||
newSessionDraft: { open: true, directoryOverride: null, parentID: null },
|
||||
|
||||
// Voice state (initialized to disconnected/idle)
|
||||
voiceStatus: 'disconnected',
|
||||
voiceMode: 'idle',
|
||||
|
||||
// Voice actions
|
||||
setVoiceStatus: (status: import("./types/sessionTypes").VoiceStatus) => {
|
||||
set({ voiceStatus: status });
|
||||
},
|
||||
setVoiceMode: (mode: import("./types/sessionTypes").VoiceMode) => {
|
||||
set({ voiceMode: mode });
|
||||
},
|
||||
|
||||
getSessionAgentEditMode: (sessionId: string, agentName: string | undefined, defaultMode?: EditPermissionMode) => {
|
||||
return useContextStore.getState().getSessionAgentEditMode(sessionId, agentName, defaultMode);
|
||||
},
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"types": ["@types/dom-speech-recognition"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"moduleDetection": "force",
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"baseUrl": ".",
|
||||
"types": ["vite/client"],
|
||||
"types": ["vite/client", "@types/dom-speech-recognition"],
|
||||
"paths": {
|
||||
"@/*": ["../ui/src/*"],
|
||||
"@vscode/*": ["./webview/*"],
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
"jsonc-parser": "^3.3.1",
|
||||
"next-themes": "^0.4.6",
|
||||
"node-pty": "^1.1.0",
|
||||
"openai": "^4.79.0",
|
||||
"qrcode-terminal": "^0.12.0",
|
||||
"react": "^19.1.1",
|
||||
"react-dom": "^19.1.1",
|
||||
|
||||
@@ -4829,7 +4829,36 @@ async function main(options = {}) {
|
||||
|
||||
console.log(`Starting OpenChamber on port ${port === 0 ? 'auto' : port}`);
|
||||
|
||||
// Check macOS Say TTS availability once at startup
|
||||
let sayTTSCapability = { available: false, voices: [], reason: 'Not checked' };
|
||||
if (process.platform === 'darwin') {
|
||||
try {
|
||||
const { exec } = await import('child_process');
|
||||
const { promisify } = await import('util');
|
||||
const execAsync = promisify(exec);
|
||||
const { stdout } = await execAsync('say -v "?"');
|
||||
const voices = stdout.split('\n')
|
||||
.filter(line => line.trim())
|
||||
.map(line => {
|
||||
const match = line.match(/^(.+?)\s+([a-zA-Z]{2}_[a-zA-Z]{2,3})\s+#/);
|
||||
if (match) {
|
||||
return { name: match[1].trim(), locale: match[2] };
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
sayTTSCapability = { available: true, voices };
|
||||
console.log(`macOS Say TTS available with ${voices.length} voices`);
|
||||
} catch (error) {
|
||||
sayTTSCapability = { available: false, voices: [], reason: 'say command not available' };
|
||||
console.log('macOS Say TTS not available:', error.message);
|
||||
}
|
||||
} else {
|
||||
sayTTSCapability = { available: false, voices: [], reason: 'Not macOS' };
|
||||
}
|
||||
|
||||
const app = express();
|
||||
app.set('trust proxy', true);
|
||||
expressApp = app;
|
||||
server = http.createServer(app);
|
||||
|
||||
@@ -4862,7 +4891,9 @@ async function main(options = {}) {
|
||||
req.path.startsWith('/api/prompts') ||
|
||||
req.path.startsWith('/api/terminal') ||
|
||||
req.path.startsWith('/api/opencode') ||
|
||||
req.path.startsWith('/api/push')
|
||||
req.path.startsWith('/api/push') ||
|
||||
req.path.startsWith('/api/voice') ||
|
||||
req.path.startsWith('/api/tts')
|
||||
) {
|
||||
|
||||
express.json({ limit: '50mb' })(req, res, next);
|
||||
@@ -5025,6 +5056,220 @@ async function main(options = {}) {
|
||||
res.json(getSessionActivitySnapshot());
|
||||
});
|
||||
|
||||
// Voice token endpoint - returns OpenAI TTS availability status
|
||||
app.post('/api/voice/token', async (req, res) => {
|
||||
console.log('[Voice] Token request received:', { body: req.body, headers: req.headers['content-type'] });
|
||||
try {
|
||||
const openaiApiKey = process.env.OPENAI_API_KEY;
|
||||
console.log('[Voice] OpenAI API Key present:', !!openaiApiKey);
|
||||
|
||||
if (!openaiApiKey) {
|
||||
return res.status(503).json({
|
||||
allowed: false,
|
||||
error: 'OpenAI voice service not configured. Set OPENAI_API_KEY environment variable.'
|
||||
});
|
||||
}
|
||||
|
||||
// Return success - OpenAI TTS is available
|
||||
res.json({
|
||||
allowed: true,
|
||||
provider: 'openai',
|
||||
message: 'OpenAI TTS is available'
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Voice] Token generation error:', error);
|
||||
res.status(500).json({
|
||||
allowed: false,
|
||||
error: 'Voice service error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Server-side TTS endpoint - streams audio from OpenAI TTS API
|
||||
app.post('/api/tts/speak', async (req, res) => {
|
||||
try {
|
||||
const { text, voice = 'nova', model = 'gpt-4o-mini-tts', speed = 0.9, instructions, summarize = false, providerId, modelId, threshold = 200, maxLength = 500, apiKey } = req.body || {};
|
||||
|
||||
console.log('[TTS] Request received:', { voice, model, speed, textLength: text?.length, hasApiKey: !!apiKey });
|
||||
|
||||
if (!text || typeof text !== 'string' || !text.trim()) {
|
||||
return res.status(400).json({ error: 'Text is required' });
|
||||
}
|
||||
|
||||
// Dynamically import the TTS service (ESM)
|
||||
const { ttsService } = await import('./lib/tts-service.js');
|
||||
|
||||
// Check availability - either server-configured or client-provided API key
|
||||
const hasServerKey = ttsService.isAvailable();
|
||||
const hasClientKey = apiKey && typeof apiKey === 'string' && apiKey.trim().length > 0;
|
||||
|
||||
if (!hasServerKey && !hasClientKey) {
|
||||
return res.status(503).json({
|
||||
error: 'TTS service not available. Please configure OpenAI in OpenCode or provide an API key in settings.'
|
||||
});
|
||||
}
|
||||
|
||||
let textToSpeak = text.trim();
|
||||
|
||||
// Optionally summarize long text before speaking using zen API
|
||||
if (summarize && textToSpeak.length > threshold) {
|
||||
try {
|
||||
const { summarizeText } = await import('./lib/summarization-service.js');
|
||||
const result = await summarizeText({ text: textToSpeak, threshold, maxLength });
|
||||
|
||||
if (result.summarized && result.summary) {
|
||||
textToSpeak = result.summary;
|
||||
}
|
||||
} catch (summarizeError) {
|
||||
console.error('[TTS/speak] Summarization failed:', summarizeError);
|
||||
// Continue with original text if summarization fails
|
||||
}
|
||||
}
|
||||
|
||||
const result = await ttsService.generateSpeechStream({
|
||||
text: textToSpeak,
|
||||
voice,
|
||||
model,
|
||||
speed,
|
||||
instructions,
|
||||
apiKey: hasClientKey ? apiKey.trim() : undefined
|
||||
});
|
||||
|
||||
// Set headers for audio streaming
|
||||
// Note: Don't set Transfer-Encoding manually - Express handles it automatically
|
||||
res.setHeader('Content-Type', result.contentType);
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
|
||||
// Collect the full audio buffer and send it
|
||||
// This avoids chunked encoding issues with proxies
|
||||
const reader = result.stream.getReader();
|
||||
const chunks = [];
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(Buffer.from(value));
|
||||
}
|
||||
const audioBuffer = Buffer.concat(chunks);
|
||||
res.setHeader('Content-Length', audioBuffer.length);
|
||||
res.send(audioBuffer);
|
||||
} catch (streamError) {
|
||||
console.error('[TTS] Stream error:', streamError);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Stream error' });
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[TTS] Error:', error);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({
|
||||
error: error instanceof Error ? error.message : 'TTS generation failed'
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Import summarization service
|
||||
const { summarizeText, sanitizeForTTS } = await import('./lib/summarization-service.js');
|
||||
|
||||
app.post('/api/tts/summarize', async (req, res) => {
|
||||
try {
|
||||
const { text, threshold = 200, maxLength = 500 } = req.body || {};
|
||||
|
||||
if (!text || typeof text !== 'string' || !text.trim()) {
|
||||
return res.status(400).json({ error: 'Text is required' });
|
||||
}
|
||||
|
||||
const result = await summarizeText({ text, threshold, maxLength });
|
||||
|
||||
return res.json(result);
|
||||
} catch (error) {
|
||||
console.error('[Summarize] Error:', error);
|
||||
const sanitized = sanitizeForTTS(req.body?.text || '');
|
||||
return res.json({ summary: sanitized, summarized: false, reason: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// TTS status endpoint
|
||||
app.get('/api/tts/status', async (_req, res) => {
|
||||
try {
|
||||
const { ttsService } = await import('./lib/tts-service.js');
|
||||
res.json({
|
||||
available: ttsService.isAvailable(),
|
||||
voices: [
|
||||
'alloy', 'ash', 'ballad', 'coral', 'echo', 'fable',
|
||||
'nova', 'onyx', 'sage', 'shimmer', 'verse', 'marin', 'cedar'
|
||||
]
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: 'Failed to check TTS status' });
|
||||
}
|
||||
});
|
||||
|
||||
// macOS 'say' command TTS status endpoint - returns cached capability from startup
|
||||
app.get('/api/tts/say/status', (_req, res) => {
|
||||
res.json(sayTTSCapability);
|
||||
});
|
||||
|
||||
// macOS 'say' command TTS speak endpoint
|
||||
app.post('/api/tts/say/speak', async (req, res) => {
|
||||
try {
|
||||
const { text, voice = 'Samantha', rate = 200 } = req.body || {};
|
||||
|
||||
if (!text || typeof text !== 'string' || !text.trim()) {
|
||||
return res.status(400).json({ error: 'Text is required' });
|
||||
}
|
||||
|
||||
// Check if we're on macOS
|
||||
if (process.platform !== 'darwin') {
|
||||
return res.status(503).json({ error: 'macOS say command not available on this platform' });
|
||||
}
|
||||
|
||||
const { exec } = await import('child_process');
|
||||
const { promisify } = await import('util');
|
||||
const fs = await import('fs');
|
||||
const os = await import('os');
|
||||
const path = await import('path');
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
// Create temp file for audio output (use m4a for browser compatibility)
|
||||
const tempDir = os.tmpdir();
|
||||
const tempFile = path.join(tempDir, `say-${Date.now()}.m4a`);
|
||||
|
||||
// Escape text for shell - escape both single quotes and double quotes
|
||||
const escapedText = text.trim().replace(/'/g, "'\\''").replace(/"/g, '\\"');
|
||||
|
||||
// Generate audio file using 'say' command
|
||||
// -o outputs to file, -r sets rate (words per minute)
|
||||
// --data-format=aac outputs as m4a which browsers can decode
|
||||
const cmd = `say -v "${voice}" -r ${rate} -o "${tempFile}" --data-format=aac '${escapedText}'`;
|
||||
console.log('[TTS-Say] Generating speech:', { textLength: text.length, voice, rate });
|
||||
|
||||
await execAsync(cmd);
|
||||
|
||||
// Read the generated audio file
|
||||
const audioBuffer = await fs.promises.readFile(tempFile);
|
||||
|
||||
// Clean up temp file
|
||||
fs.promises.unlink(tempFile).catch(() => {});
|
||||
|
||||
// Send audio response
|
||||
res.setHeader('Content-Type', 'audio/mp4');
|
||||
res.setHeader('Content-Length', audioBuffer.length);
|
||||
res.send(audioBuffer);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[TTS-Say] Error:', error);
|
||||
res.status(500).json({
|
||||
error: error instanceof Error ? error.message : 'Say command failed'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// New authoritative session status endpoints
|
||||
// Server maintains the source of truth, clients only query
|
||||
|
||||
@@ -8284,7 +8529,6 @@ async function main(options = {}) {
|
||||
|
||||
const base = typeof req.body?.base === 'string' ? req.body.base.trim() : '';
|
||||
const head = typeof req.body?.head === 'string' ? req.body.head.trim() : '';
|
||||
const context = typeof req.body?.context === 'string' ? req.body.context.trim() : '';
|
||||
if (!base || !head) {
|
||||
return res.status(400).json({ error: 'base and head are required' });
|
||||
}
|
||||
@@ -8303,6 +8547,7 @@ async function main(options = {}) {
|
||||
}
|
||||
|
||||
const diffSummaries = diffs.map(({ path, diff }) => `FILE: ${path}\n${diff}`).join('\n\n');
|
||||
const context = typeof req.body?.context === 'string' ? req.body.context.trim() : '';
|
||||
|
||||
let prompt = `You are drafting a GitHub Pull Request title + description for a squash-merge workflow.
|
||||
Respond in JSON of the shape {"title": string, "body": string} (ONLY JSON in response, no markdown fences) with these rules:
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Text Summarization Service
|
||||
*
|
||||
* Uses the opencode.ai zen API with gpt-5-nano for fast, lightweight summarization.
|
||||
* Used by all TTS implementations (Browser, Say, OpenAI).
|
||||
*/
|
||||
|
||||
function buildSummarizationPrompt(maxLength) {
|
||||
return `You are a text summarizer for text-to-speech output. Create a concise, natural-sounding summary that captures the key points. Keep the summary under ${maxLength} characters.
|
||||
|
||||
CRITICAL INSTRUCTIONS:
|
||||
1. Output ONLY the final summary - no thinking, no reasoning, no explanations
|
||||
2. Do not show your work or thought process
|
||||
3. Do not use any special characters, markdown, code, URLs, file paths, or formatting
|
||||
4. Do not include phrases like "Here's a summary" or "In summary"
|
||||
5. Just provide clean, speakable text that can be read aloud
|
||||
6. Stay within the ${maxLength} character limit
|
||||
|
||||
Your response should be ready to speak immediately.`;
|
||||
}
|
||||
|
||||
const SUMMARIZE_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Sanitize text for TTS output
|
||||
* Removes markdown, URLs, file paths, and other non-speakable content
|
||||
*/
|
||||
export function sanitizeForTTS(text) {
|
||||
if (!text || typeof text !== 'string') return '';
|
||||
|
||||
return text
|
||||
// Remove markdown formatting
|
||||
.replace(/[*_~`#]/g, '')
|
||||
// Remove code blocks
|
||||
.replace(/```[\s\S]*?```/g, '')
|
||||
.replace(/`[^`]*`/g, '')
|
||||
// Remove shell-like command patterns
|
||||
.replace(/^\s*[$#>]\s*/gm, '')
|
||||
// Remove common shell operators
|
||||
.replace(/[|&;<>]/g, ' ')
|
||||
// Remove backslashes (escape characters)
|
||||
.replace(/\\/g, '')
|
||||
// Remove brackets that might be interpreted specially
|
||||
.replace(/[[\]{}()]/g, '')
|
||||
// Remove quotes that might cause issues
|
||||
.replace(/["']/g, '')
|
||||
// Remove URLs
|
||||
.replace(/https?:\/\/[^\s]+/g, ' a link ')
|
||||
// Remove file paths
|
||||
.replace(/\/[\w\-./]+/g, '')
|
||||
// Collapse multiple spaces/newlines
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text from zen API response
|
||||
*/
|
||||
function extractZenOutputText(data) {
|
||||
if (!data || typeof data !== 'object') return null;
|
||||
const output = data.output;
|
||||
if (!Array.isArray(output)) return null;
|
||||
|
||||
const messageItem = output.find(
|
||||
(item) => item && typeof item === 'object' && item.type === 'message'
|
||||
);
|
||||
if (!messageItem) return null;
|
||||
|
||||
const content = messageItem.content;
|
||||
if (!Array.isArray(content)) return null;
|
||||
|
||||
const textItem = content.find(
|
||||
(item) => item && typeof item === 'object' && item.type === 'output_text'
|
||||
);
|
||||
|
||||
const text = typeof textItem?.text === 'string' ? textItem.text.trim() : '';
|
||||
return text || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarize text using the opencode.ai zen API with gpt-5-nano
|
||||
*
|
||||
* @param {Object} options
|
||||
* @param {string} options.text - The text to summarize
|
||||
* @param {number} options.threshold - Character threshold (don't summarize if under this length)
|
||||
* @param {number} options.maxLength - Maximum character length for the summary output (50-2000)
|
||||
* @returns {Promise<{summary: string, summarized: boolean, reason?: string}>}
|
||||
*/
|
||||
export async function summarizeText({
|
||||
text,
|
||||
threshold = 200,
|
||||
maxLength = 500,
|
||||
}) {
|
||||
// Don't summarize if text is under threshold
|
||||
if (!text || text.length <= threshold) {
|
||||
return {
|
||||
summary: sanitizeForTTS(text || ''),
|
||||
summarized: false,
|
||||
reason: text ? 'Text under threshold' : 'No text provided',
|
||||
};
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), SUMMARIZE_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const prompt = buildSummarizationPrompt(maxLength);
|
||||
|
||||
const response = await fetch('https://opencode.ai/zen/v1/responses', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'gpt-5-nano',
|
||||
input: [
|
||||
{ role: 'user', content: `${prompt}\n\nText to summarize:\n${text}` },
|
||||
],
|
||||
stream: false,
|
||||
reasoning: { effort: 'low' },
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.json().catch(() => ({}));
|
||||
console.error('[Summarize] zen API error:', response.status, errorBody);
|
||||
return {
|
||||
summary: sanitizeForTTS(text),
|
||||
summarized: false,
|
||||
reason: `zen API returned ${response.status}`,
|
||||
};
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const summary = extractZenOutputText(data);
|
||||
|
||||
if (summary) {
|
||||
const sanitized = sanitizeForTTS(summary);
|
||||
return {
|
||||
summary: sanitized,
|
||||
summarized: true,
|
||||
originalLength: text.length,
|
||||
summaryLength: sanitized.length,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
summary: sanitizeForTTS(text),
|
||||
summarized: false,
|
||||
reason: 'No response from model',
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
console.error('[Summarize] Request timed out');
|
||||
return {
|
||||
summary: sanitizeForTTS(text),
|
||||
summarized: false,
|
||||
reason: 'Request timed out',
|
||||
};
|
||||
}
|
||||
console.error('[Summarize] Error:', error);
|
||||
return {
|
||||
summary: sanitizeForTTS(text),
|
||||
summarized: false,
|
||||
reason: error.message,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* Server-side Text-to-Speech Service
|
||||
*
|
||||
* Uses OpenAI's TTS API to generate audio on the server and stream it to clients.
|
||||
* This bypasses mobile Safari's audio context restrictions.
|
||||
*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { readAuthFile } from './opencode-auth.js';
|
||||
|
||||
// Voice options from OpenAI
|
||||
export const TTS_VOICES = [
|
||||
'alloy', 'ash', 'ballad', 'coral', 'echo', 'fable',
|
||||
'nova', 'onyx', 'sage', 'shimmer', 'verse', 'marin', 'cedar'
|
||||
];
|
||||
|
||||
function getOpenAIApiKey() {
|
||||
// First check environment variable
|
||||
const envKey = process.env.OPENAI_API_KEY;
|
||||
if (envKey) {
|
||||
return envKey;
|
||||
}
|
||||
|
||||
// Then check opencode auth file (same as usage tracker)
|
||||
try {
|
||||
const auth = readAuthFile();
|
||||
// Check for openai, codex, or chatgpt aliases
|
||||
const openaiAuth = auth.openai || auth.codex || auth.chatgpt;
|
||||
if (openaiAuth) {
|
||||
// Handle both string format (just the token) and object format
|
||||
if (typeof openaiAuth === 'string') {
|
||||
return openaiAuth;
|
||||
}
|
||||
// Try access token first (OAuth), then regular token
|
||||
if (openaiAuth.access) {
|
||||
return openaiAuth.access;
|
||||
}
|
||||
if (openaiAuth.token) {
|
||||
return openaiAuth.token;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('[TTSService] Failed to read auth file:', error.message);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
class TTSService {
|
||||
constructor() {
|
||||
this._client = null;
|
||||
this._lastApiKey = null;
|
||||
}
|
||||
|
||||
_getClient() {
|
||||
const apiKey = getOpenAIApiKey();
|
||||
|
||||
// If API key changed or client doesn't exist, create new client
|
||||
if (apiKey && (!this._client || this._lastApiKey !== apiKey)) {
|
||||
this._client = new OpenAI({ apiKey });
|
||||
this._lastApiKey = apiKey;
|
||||
}
|
||||
|
||||
return this._client;
|
||||
}
|
||||
|
||||
isAvailable() {
|
||||
return this._getClient() !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate speech and return as a stream
|
||||
*/
|
||||
async generateSpeechStream(options) {
|
||||
const {
|
||||
text,
|
||||
voice = 'coral',
|
||||
model = 'gpt-4o-mini-tts',
|
||||
speed = 1.0,
|
||||
instructions,
|
||||
apiKey
|
||||
} = options;
|
||||
|
||||
// Use provided API key or fall back to configured key
|
||||
let client;
|
||||
if (apiKey) {
|
||||
client = new OpenAI({ apiKey });
|
||||
} else {
|
||||
client = this._getClient();
|
||||
}
|
||||
|
||||
if (!client) {
|
||||
throw new Error('OpenAI API key not configured. Set OPENAI_API_KEY environment variable, configure OpenAI in OpenCode, or provide an API key in settings.');
|
||||
}
|
||||
|
||||
if (!text.trim()) {
|
||||
throw new Error('Text is required for TTS');
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[TTSService] Generating speech with voice:', voice, 'model:', model);
|
||||
const response = await client.audio.speech.create({
|
||||
model,
|
||||
voice,
|
||||
input: text,
|
||||
speed,
|
||||
...(instructions && { instructions }),
|
||||
response_format: 'mp3',
|
||||
});
|
||||
|
||||
// Convert the response to a web stream
|
||||
const stream = response.body;
|
||||
|
||||
return {
|
||||
stream,
|
||||
contentType: 'audio/mpeg',
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[TTSService] Error generating speech:', error);
|
||||
throw new Error(`Failed to generate speech: ${error.message || 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate speech and return as a buffer (for caching)
|
||||
*/
|
||||
async generateSpeechBuffer(options) {
|
||||
const client = this._getClient();
|
||||
if (!client) {
|
||||
throw new Error('OpenAI API key not configured. Set OPENAI_API_KEY environment variable or configure OpenAI in OpenCode.');
|
||||
}
|
||||
|
||||
const {
|
||||
text,
|
||||
voice = 'coral',
|
||||
model = 'gpt-4o-mini-tts',
|
||||
speed = 1.0,
|
||||
instructions
|
||||
} = options;
|
||||
|
||||
try {
|
||||
const response = await client.audio.speech.create({
|
||||
model,
|
||||
voice,
|
||||
input: text,
|
||||
speed,
|
||||
...(instructions && { instructions }),
|
||||
response_format: 'mp3',
|
||||
});
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return Buffer.from(arrayBuffer);
|
||||
} catch (error) {
|
||||
console.error('[TTSService] Error generating speech buffer:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const ttsService = new TTSService();
|
||||
export { TTSService };
|
||||
@@ -13,7 +13,7 @@
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"baseUrl": ".",
|
||||
"types": ["vite/client"],
|
||||
"types": ["vite/client", "@types/dom-speech-recognition"],
|
||||
"paths": {
|
||||
"@/*": ["../ui/src/*", "./src/*"],
|
||||
"@web/*": ["./src/*"],
|
||||
|
||||
Reference in New Issue
Block a user