feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)
Complete rebuild of voice input on a server-authoritative streaming architecture, replacing the legacy Web Speech / whole-blob / WASM engines and the dead voice-agent layer (~4k lines removed). Speech-to-text (dictation): - Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with seq/ack ordering; buffered audio is retained and replayed on reconnect - Server transcribes and streams live partial transcripts back; segments auto-commit every ~15s with silence suppression and adaptive finalization timeouts - Local provider (default, zero config): sherpa-onnx models in a forked worker process — auto-download with progress, staged extraction with verification, corrupt-model auto-recovery, idle shutdown after 5 min - Model catalog with settings picker (accuracy/speed ratings, sizes, download/delete): Parakeet TDT v2 (English) and v3 (25 European languages, auto-detected), Whisper base and tiny (multilingual, light) - OpenAI-compatible provider for any Whisper endpoint - Composer overlay with live transcript, volume meter, timer, and cancel / insert / insert-and-send actions; failed transcriptions keep their audio for retry or accepting the partial text as-is - Configurable keyboard shortcut (default mod+alt+v) toggles dictation; Enter confirms and Escape cancels while recording - Overlay is pixel-aligned with the composer (measured footer height, matching paddings/typography/gaps) — no layout shift when toggling Text-to-speech: - Local Kokoro provider (English, 11 voices) synthesized in the same worker via /api/dictation/tts/speak, managed by the shared model pipeline; sentence-pipelined playback keeps time-to-first-audio at ~1 sentence regardless of message length, and stop cancels in-flight synthesis - Sanitizer keeps inline-code content (strips backticks only), reads interword slashes aloud, and removes only absolute file paths Settings: - Voice page unified: a single read-aloud toggle owns all playback options (the confusing "Enable Voice Mode" is gone); a new "Enable voice input" toggle (default on, persisted to settings.json) hides the composer mic entirely when disabled Mobile and transport: - iOS/Android microphone permissions added (dictation was previously impossible on mobile) - Fixed Android WebSocket upgrades: the Capacitor WebView origin (https://localhost) was missing from the packaged-client allowlist, 403-ing every WS connection — root cause of the old mobile SSE lock, which is now removed for all transports Security and conventions: - All HTTP routes sit behind the global /api auth gate; the WS upgrade explicitly validates the UI session and origin, with oc_url_token narrowly allowlisted and covered by tests; the dictation socket mints a fresh URL token before connecting - Routes register before the generic OpenCode proxy; the client goes through runtimeFetch/getRuntimeUrlResolver, and runtime switches reset the dictation socket - VS Code deliberately reports dictation as unavailable (no server process in that runtime) CI: workflow Node bumped 20 -> 22 to match the repo engines and fix better-sqlite3 installs broken by node-gyp@latest on Node 20. New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64, Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
This commit is contained in:
Bohdan Triapitsyn
committed by
GitHub
GitHub
parent
3f5151d424
commit
de1b85ac56
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,408 @@
|
||||
/**
|
||||
* Streaming dictation state machine.
|
||||
*
|
||||
* Status flow: idle -> recording -> uploading -> idle | failed.
|
||||
* While recording, mic PCM chunks stream to the server, which sends back live
|
||||
* partial transcripts. Confirm finalizes and resolves the full text; failed
|
||||
* dictations retain their audio segments so retry can replay them.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
|
||||
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
|
||||
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
|
||||
|
||||
export interface UseDictationOptions {
|
||||
onTranscript?: (text: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
canStart?: () => boolean;
|
||||
}
|
||||
|
||||
export interface UseDictationResult {
|
||||
status: DictationStatus;
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
partialTranscript: string;
|
||||
volume: number;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
errorReason: string | null;
|
||||
startDictation: () => Promise<void>;
|
||||
confirmDictation: () => Promise<string | null>;
|
||||
cancelDictation: () => Promise<void>;
|
||||
retryFailedDictation: () => Promise<string | null>;
|
||||
acceptPartialTranscript: () => string | null;
|
||||
discardFailedDictation: () => void;
|
||||
}
|
||||
|
||||
const DURATION_TICK_MS = 1000;
|
||||
|
||||
const toError = (value: unknown): Error =>
|
||||
value instanceof Error ? value : new Error(String(value));
|
||||
|
||||
const getDictationStartOptions = (): DictationStartOptions => {
|
||||
const state = useConfigStore.getState();
|
||||
const language = state.sttLanguage?.trim();
|
||||
if (state.sttProvider === 'openai-compatible') {
|
||||
return {
|
||||
provider: 'openai-compatible',
|
||||
...(language ? { language } : {}),
|
||||
openaiCompatible: {
|
||||
baseUrl: state.sttServerUrl,
|
||||
model: state.sttModel,
|
||||
...(state.sttApiKey ? { apiKey: state.sttApiKey } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
provider: 'local',
|
||||
...(language ? { language } : {}),
|
||||
localModel: state.sttLocalModel,
|
||||
};
|
||||
};
|
||||
|
||||
export function useDictation(options: UseDictationOptions = {}): UseDictationResult {
|
||||
const { onTranscript, onError, canStart } = options;
|
||||
|
||||
const [status, setStatus] = useState<DictationStatus>('idle');
|
||||
const [partialTranscript, setPartialTranscript] = useState('');
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [errorReason, setErrorReason] = useState<string | null>(null);
|
||||
|
||||
const statusRef = useRef(status);
|
||||
useEffect(() => {
|
||||
statusRef.current = status;
|
||||
}, [status]);
|
||||
|
||||
const latestPartialRef = useRef('');
|
||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const actionGateRef = useRef({ starting: false, confirming: false, cancelling: false });
|
||||
|
||||
const onTranscriptRef = useRef(onTranscript);
|
||||
const onErrorRef = useRef(onError);
|
||||
useEffect(() => {
|
||||
onTranscriptRef.current = onTranscript;
|
||||
onErrorRef.current = onError;
|
||||
}, [onTranscript, onError]);
|
||||
|
||||
const senderRef = useRef<DictationStreamSender | null>(null);
|
||||
if (!senderRef.current) {
|
||||
senderRef.current = new DictationStreamSender({
|
||||
client: dictationClient,
|
||||
getStartOptions: getDictationStartOptions,
|
||||
});
|
||||
}
|
||||
|
||||
const stopDurationTracking = useCallback(() => {
|
||||
if (durationIntervalRef.current) {
|
||||
clearInterval(durationIntervalRef.current);
|
||||
durationIntervalRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const startDurationTracking = useCallback(() => {
|
||||
if (durationIntervalRef.current) {
|
||||
return;
|
||||
}
|
||||
durationIntervalRef.current = setInterval(() => {
|
||||
setDuration((prev) => prev + 1);
|
||||
}, DURATION_TICK_MS);
|
||||
}, []);
|
||||
|
||||
const reportError = useCallback((err: unknown) => {
|
||||
const normalized = toError(err);
|
||||
setError(normalized.message);
|
||||
const reason = (normalized as Error & { reasonCode?: string }).reasonCode;
|
||||
setErrorReason(typeof reason === 'string' ? reason : null);
|
||||
onErrorRef.current?.(normalized);
|
||||
}, []);
|
||||
|
||||
const clearError = useCallback(() => {
|
||||
setError(null);
|
||||
setErrorReason(null);
|
||||
}, []);
|
||||
|
||||
const clearStreamingState = useCallback(() => {
|
||||
senderRef.current?.clearAll();
|
||||
latestPartialRef.current = '';
|
||||
setPartialTranscript('');
|
||||
}, []);
|
||||
|
||||
// Live partial transcripts for the active dictation.
|
||||
useEffect(() => {
|
||||
return dictationClient.onPartial((dictationId, text) => {
|
||||
const activeDictationId = senderRef.current?.getDictationId();
|
||||
if (!activeDictationId || dictationId !== activeDictationId) {
|
||||
return;
|
||||
}
|
||||
latestPartialRef.current = text;
|
||||
setPartialTranscript(text);
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Restart the stream (replaying buffered segments) after a reconnect.
|
||||
useEffect(() => {
|
||||
return dictationClient.subscribeConnectionStatus((connected) => {
|
||||
if (!connected) {
|
||||
return;
|
||||
}
|
||||
if (statusRef.current !== 'recording') {
|
||||
return;
|
||||
}
|
||||
void senderRef.current?.restartStream().catch((err) => {
|
||||
reportError(err);
|
||||
});
|
||||
});
|
||||
}, [reportError]);
|
||||
|
||||
const audio = useDictationAudioSource({
|
||||
onPcmSegment: (audioData) => {
|
||||
senderRef.current?.enqueueSegment(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
onErrorRef.current?.(err);
|
||||
},
|
||||
});
|
||||
const audioStopRef = useRef(audio.stop);
|
||||
useEffect(() => {
|
||||
audioStopRef.current = audio.stop;
|
||||
}, [audio.stop]);
|
||||
|
||||
const handleSuccess = useCallback(
|
||||
(text: string): string | null => {
|
||||
setDuration(0);
|
||||
setStatus('idle');
|
||||
const transcriptText = text.trim().length > 0 ? text.trim() : latestPartialRef.current.trim();
|
||||
clearStreamingState();
|
||||
if (!transcriptText) {
|
||||
return null;
|
||||
}
|
||||
onTranscriptRef.current?.(transcriptText);
|
||||
return transcriptText;
|
||||
},
|
||||
[clearStreamingState],
|
||||
);
|
||||
|
||||
const handleFailure = useCallback(
|
||||
(failure: unknown) => {
|
||||
if (senderRef.current?.hasSegments()) {
|
||||
setStatus('failed');
|
||||
} else {
|
||||
setStatus('idle');
|
||||
}
|
||||
reportError(failure);
|
||||
},
|
||||
[reportError],
|
||||
);
|
||||
|
||||
const startDictation = useCallback(async () => {
|
||||
const gate = actionGateRef.current;
|
||||
if (gate.starting || gate.confirming || gate.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (statusRef.current !== 'idle') {
|
||||
return;
|
||||
}
|
||||
if (canStart && !canStart()) {
|
||||
return;
|
||||
}
|
||||
|
||||
gate.starting = true;
|
||||
clearError();
|
||||
setPartialTranscript('');
|
||||
setDuration(0);
|
||||
setStatus('recording');
|
||||
statusRef.current = 'recording';
|
||||
clearStreamingState();
|
||||
|
||||
try {
|
||||
await audio.start();
|
||||
startDurationTracking();
|
||||
// Open the stream eagerly so partials start flowing immediately.
|
||||
await senderRef.current?.restartStream().catch((err) => {
|
||||
// Non-fatal: segments buffer locally and finish() retries the
|
||||
// start, but surface the reason (e.g. model downloading) so
|
||||
// the overlay can show it.
|
||||
reportError(err);
|
||||
});
|
||||
} catch (err) {
|
||||
await audio.stop().catch(() => undefined);
|
||||
stopDurationTracking();
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
reportError(err);
|
||||
} finally {
|
||||
gate.starting = false;
|
||||
}
|
||||
}, [audio, canStart, clearError, clearStreamingState, reportError, startDurationTracking, stopDurationTracking]);
|
||||
|
||||
const cancelDictation = useCallback(async () => {
|
||||
const gate = actionGateRef.current;
|
||||
if (gate.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (statusRef.current !== 'recording' && statusRef.current !== 'uploading') {
|
||||
return;
|
||||
}
|
||||
gate.cancelling = true;
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
clearError();
|
||||
|
||||
// Optimistic: dismiss the overlay immediately. Tearing down the audio
|
||||
// graph (AudioContext.close) can take up to ~1s on mobile WebViews and
|
||||
// must not delay the visible response to Cancel.
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
try {
|
||||
senderRef.current?.cancel();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
clearStreamingState();
|
||||
|
||||
try {
|
||||
await audio.stop();
|
||||
} catch {
|
||||
// Cancelled anyway; mic teardown failures are not user-actionable.
|
||||
} finally {
|
||||
gate.cancelling = false;
|
||||
}
|
||||
}, [audio, clearError, clearStreamingState, stopDurationTracking]);
|
||||
|
||||
const confirmDictation = useCallback(async (): Promise<string | null> => {
|
||||
const gate = actionGateRef.current;
|
||||
if (gate.confirming) {
|
||||
return null;
|
||||
}
|
||||
if (statusRef.current !== 'recording') {
|
||||
return null;
|
||||
}
|
||||
|
||||
gate.confirming = true;
|
||||
clearError();
|
||||
stopDurationTracking();
|
||||
|
||||
try {
|
||||
await audio.stop();
|
||||
setStatus('uploading');
|
||||
statusRef.current = 'uploading';
|
||||
|
||||
const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
|
||||
if (finalSeq < 0) {
|
||||
return handleSuccess('');
|
||||
}
|
||||
|
||||
const result = await senderRef.current!.finish(finalSeq);
|
||||
return handleSuccess(result.text);
|
||||
} catch (err) {
|
||||
handleFailure(err);
|
||||
return null;
|
||||
} finally {
|
||||
gate.confirming = false;
|
||||
}
|
||||
}, [audio, clearError, handleFailure, handleSuccess, stopDurationTracking]);
|
||||
|
||||
const retryFailedDictation = useCallback(async (): Promise<string | null> => {
|
||||
if (statusRef.current !== 'failed' || !senderRef.current?.hasSegments()) {
|
||||
return null;
|
||||
}
|
||||
clearError();
|
||||
setStatus('uploading');
|
||||
statusRef.current = 'uploading';
|
||||
|
||||
try {
|
||||
senderRef.current.resetStreamForReplay();
|
||||
const finalSeq = senderRef.current.getFinalSeq();
|
||||
const result = await senderRef.current.finish(finalSeq);
|
||||
return handleSuccess(result.text);
|
||||
} catch (err) {
|
||||
handleFailure(err);
|
||||
return null;
|
||||
}
|
||||
}, [clearError, handleFailure, handleSuccess]);
|
||||
|
||||
/**
|
||||
* Failed dictations still hold the last streamed partial transcript.
|
||||
* Accept it as-is instead of retrying the full transcription.
|
||||
*/
|
||||
const acceptPartialTranscript = useCallback((): string | null => {
|
||||
if (statusRef.current !== 'failed') {
|
||||
return null;
|
||||
}
|
||||
const text = latestPartialRef.current.trim();
|
||||
setDuration(0);
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
clearError();
|
||||
try {
|
||||
senderRef.current?.cancel();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
clearStreamingState();
|
||||
if (!text) {
|
||||
return null;
|
||||
}
|
||||
onTranscriptRef.current?.(text);
|
||||
return text;
|
||||
}, [clearError, clearStreamingState]);
|
||||
|
||||
const discardFailedDictation = useCallback(() => {
|
||||
setDuration(0);
|
||||
setStatus('idle');
|
||||
statusRef.current = 'idle';
|
||||
clearError();
|
||||
clearStreamingState();
|
||||
}, [clearError, clearStreamingState]);
|
||||
|
||||
// While recording without an open stream (e.g. the model is still
|
||||
// downloading), retry the stream start so live partials kick in as soon
|
||||
// as the provider becomes ready. Buffered segments replay on success.
|
||||
useEffect(() => {
|
||||
if (status !== 'recording' || errorReason !== 'model_download_in_progress') {
|
||||
return;
|
||||
}
|
||||
const interval = setInterval(() => {
|
||||
if (statusRef.current !== 'recording' || senderRef.current?.getDictationId()) {
|
||||
return;
|
||||
}
|
||||
senderRef.current?.restartStream().then(() => {
|
||||
clearError();
|
||||
}).catch((err) => {
|
||||
reportError(err);
|
||||
});
|
||||
}, 3000);
|
||||
return () => clearInterval(interval);
|
||||
}, [status, errorReason, clearError, reportError]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stopDurationTracking();
|
||||
void audioStopRef.current().catch(() => undefined);
|
||||
senderRef.current?.cancel();
|
||||
};
|
||||
}, [stopDurationTracking]);
|
||||
|
||||
return {
|
||||
status,
|
||||
isRecording: status === 'recording',
|
||||
isProcessing: status === 'uploading',
|
||||
partialTranscript,
|
||||
volume: audio.volume,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
startDictation,
|
||||
confirmDictation,
|
||||
cancelDictation,
|
||||
retryFailedDictation,
|
||||
acceptPartialTranscript,
|
||||
discardFailedDictation,
|
||||
};
|
||||
}
|
||||
@@ -474,6 +474,18 @@ export const useKeyboardShortcuts = () => {
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventMatchesShortcut(e, combo('toggle_dictation'))) {
|
||||
const { activeMainTab, isCommandPaletteOpen, isHelpDialogOpen, isSessionSwitcherOpen, isSettingsDialogOpen } = useUIStore.getState();
|
||||
if (activeMainTab !== 'chat' || isCommandPaletteOpen || isHelpDialogOpen || isSessionSwitcherOpen || isSettingsDialogOpen) {
|
||||
return;
|
||||
}
|
||||
e.preventDefault();
|
||||
// Dictation state lives inside the composer's isolated component;
|
||||
// toggle it via an event instead of subscribing this hot hook to it.
|
||||
window.dispatchEvent(new CustomEvent('openchamber:dictation-toggle'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.key === 'Escape') {
|
||||
const target = e.target as Element | null;
|
||||
const isInsideDialog = Boolean(target?.closest('[role="dialog"]'));
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* useLocalTTS Hook
|
||||
*
|
||||
* React hook for local (Kokoro via sherpa-onnx) text-to-speech playback.
|
||||
* Synthesis runs on the OpenChamber server in the dictation worker.
|
||||
*
|
||||
* Long texts are pipelined by sentence chunks: the first chunk starts playing
|
||||
* as soon as it is synthesized while the next chunk synthesizes in the
|
||||
* background, so time-to-first-audio stays ~1 chunk regardless of message
|
||||
* length.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
|
||||
export interface LocalTTSSpeakOptions {
|
||||
/** Kokoro speaker id (0-10) */
|
||||
speakerId?: number;
|
||||
/** Playback speed multiplier (1.0 = normal) */
|
||||
speed?: number;
|
||||
onStart?: () => void;
|
||||
onEnd?: () => void;
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
export interface UseLocalTTSReturn {
|
||||
isPlaying: boolean;
|
||||
error: string | null;
|
||||
speak: (text: string, options?: LocalTTSSpeakOptions) => Promise<void>;
|
||||
stop: () => void;
|
||||
/** Unlock audio for mobile Safari - call this on user gesture */
|
||||
unlockAudio: () => Promise<void>;
|
||||
}
|
||||
|
||||
/** Target chunk size: big enough to amortize requests, small enough for low latency. */
|
||||
const MIN_CHUNK_CHARS = 60;
|
||||
const MAX_CHUNK_CHARS = 400;
|
||||
|
||||
/**
|
||||
* Split text into sentence-aligned chunks for pipelined synthesis.
|
||||
* Sentences are merged until MIN_CHUNK_CHARS and hard-split at
|
||||
* MAX_CHUNK_CHARS so a single run-on sentence cannot stall the pipeline.
|
||||
*/
|
||||
export function splitTextForSynthesis(text: string): string[] {
|
||||
const normalized = text.replace(/\s+/g, ' ').trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const sentences = normalized.match(/[^.!?…]+[.!?…]+["')\]]*\s*|[^.!?…]+$/g) ?? [normalized];
|
||||
|
||||
const chunks: string[] = [];
|
||||
let current = '';
|
||||
for (const sentence of sentences) {
|
||||
for (let offset = 0; offset < sentence.length; offset += MAX_CHUNK_CHARS) {
|
||||
const piece = sentence.slice(offset, offset + MAX_CHUNK_CHARS);
|
||||
if (current && current.length + piece.length > MAX_CHUNK_CHARS) {
|
||||
chunks.push(current.trim());
|
||||
current = '';
|
||||
}
|
||||
current += piece;
|
||||
if (current.length >= MIN_CHUNK_CHARS && /[.!?…]["')\]]*\s*$/.test(current)) {
|
||||
chunks.push(current.trim());
|
||||
current = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
if (current.trim()) {
|
||||
chunks.push(current.trim());
|
||||
}
|
||||
|
||||
// Latency trim for the FIRST chunk only: if the opening sentence is long,
|
||||
// split it at a clause boundary (comma/semicolon/colon) so the first
|
||||
// audio starts sooner. Only the first chunk gets this treatment — comma
|
||||
// splits inside every sentence would chop the prosody, and after playback
|
||||
// starts the pipeline hides synthesis time anyway.
|
||||
if (chunks.length > 0 && chunks[0].length > 120) {
|
||||
const first = chunks[0];
|
||||
const clauseBreak = /[,;:]\s/g;
|
||||
let bestBreak = -1;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = clauseBreak.exec(first)) !== null) {
|
||||
const end = match.index + 1;
|
||||
if (end < 40) {
|
||||
continue;
|
||||
}
|
||||
if (end > 160) {
|
||||
break;
|
||||
}
|
||||
bestBreak = end;
|
||||
break;
|
||||
}
|
||||
if (bestBreak > 0) {
|
||||
const head = first.slice(0, bestBreak).trim();
|
||||
const tail = first.slice(bestBreak).trim();
|
||||
chunks.splice(0, 1, head, tail);
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
let sharedAudioContext: AudioContext | null = null;
|
||||
|
||||
function getAudioContext(): AudioContext {
|
||||
if (!sharedAudioContext) {
|
||||
sharedAudioContext = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
}
|
||||
return sharedAudioContext;
|
||||
}
|
||||
|
||||
interface PlaybackSession {
|
||||
cancelled: boolean;
|
||||
abort: AbortController;
|
||||
}
|
||||
|
||||
export function useLocalTTS(): UseLocalTTSReturn {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const audioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const sessionRef = useRef<PlaybackSession | null>(null);
|
||||
|
||||
const unlockAudio = useCallback(async (): Promise<void> => {
|
||||
try {
|
||||
const ctx = getAudioContext();
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
}
|
||||
const buffer = ctx.createBuffer(1, 1, 22050);
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(ctx.destination);
|
||||
source.start(0);
|
||||
} catch {
|
||||
// Unlocking is best-effort.
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
const session = sessionRef.current;
|
||||
if (session) {
|
||||
session.cancelled = true;
|
||||
session.abort.abort();
|
||||
sessionRef.current = null;
|
||||
}
|
||||
if (audioSourceRef.current) {
|
||||
try {
|
||||
audioSourceRef.current.onended = null;
|
||||
audioSourceRef.current.stop();
|
||||
} catch {
|
||||
// Already stopped
|
||||
}
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
setIsPlaying(false);
|
||||
}, []);
|
||||
|
||||
const speak = useCallback(async (text: string, options?: LocalTTSSpeakOptions): Promise<void> => {
|
||||
stop();
|
||||
|
||||
const chunks = splitTextForSynthesis(text);
|
||||
if (chunks.length === 0) {
|
||||
setError('No text to speak');
|
||||
options?.onError?.('No text to speak');
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
|
||||
const session: PlaybackSession = { cancelled: false, abort: new AbortController() };
|
||||
sessionRef.current = session;
|
||||
|
||||
const fetchChunk = async (chunk: string): Promise<ArrayBuffer> => {
|
||||
const response = await runtimeFetch('/api/dictation/tts/speak', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
text: chunk,
|
||||
...(typeof options?.speakerId === 'number' ? { speakerId: options.speakerId } : {}),
|
||||
...(typeof options?.speed === 'number' ? { speed: options.speed } : {}),
|
||||
}),
|
||||
signal: session.abort.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
|
||||
throw new Error(errorData.error || `HTTP ${response.status}`);
|
||||
}
|
||||
return (await response.blob()).arrayBuffer();
|
||||
};
|
||||
|
||||
const playBuffer = async (arrayBuffer: ArrayBuffer): Promise<void> => {
|
||||
const ctx = getAudioContext();
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
}
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
if (session.cancelled) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(ctx.destination);
|
||||
audioSourceRef.current = source;
|
||||
source.onended = () => {
|
||||
if (audioSourceRef.current === source) {
|
||||
audioSourceRef.current = null;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
source.start(0);
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
setIsPlaying(true);
|
||||
options?.onStart?.();
|
||||
|
||||
// Pipelined: synthesize chunk N+1 while chunk N is playing.
|
||||
let nextFetch: Promise<ArrayBuffer> = fetchChunk(chunks[0]);
|
||||
for (let i = 0; i < chunks.length; i += 1) {
|
||||
const buffer = await nextFetch;
|
||||
if (session.cancelled) {
|
||||
return;
|
||||
}
|
||||
if (i + 1 < chunks.length) {
|
||||
nextFetch = fetchChunk(chunks[i + 1]);
|
||||
}
|
||||
await playBuffer(buffer);
|
||||
if (session.cancelled) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
options?.onEnd?.();
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError' || session.cancelled) {
|
||||
return;
|
||||
}
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to speak';
|
||||
setError(errorMsg);
|
||||
options?.onError?.(errorMsg);
|
||||
setIsPlaying(false);
|
||||
} finally {
|
||||
if (sessionRef.current === session) {
|
||||
sessionRef.current = null;
|
||||
}
|
||||
}
|
||||
}, [stop]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
isPlaying,
|
||||
error,
|
||||
speak,
|
||||
stop,
|
||||
unlockAudio,
|
||||
};
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { useCallback, useState } from 'react';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
import { useServerTTS } from './useServerTTS';
|
||||
import { useSayTTS } from './useSayTTS';
|
||||
import { useLocalTTS } from './useLocalTTS';
|
||||
import { browserVoiceService } from '@/lib/voice/browserVoiceService';
|
||||
import { sanitizeForTTS } from '@/lib/voice/summarize';
|
||||
|
||||
@@ -29,6 +30,7 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
const speechPitch = useConfigStore((state) => state.speechPitch);
|
||||
const speechVolume = useConfigStore((state) => state.speechVolume);
|
||||
const sayVoice = useConfigStore((state) => state.sayVoice);
|
||||
const localTtsVoiceId = useConfigStore((state) => state.localTtsVoiceId);
|
||||
const browserVoice = useConfigStore((state) => state.browserVoice);
|
||||
const openaiVoice = useConfigStore((state) => state.openaiVoice);
|
||||
const openaiCompatibleVoice = useConfigStore((state) => state.openaiCompatibleVoice);
|
||||
@@ -48,13 +50,15 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
const { speak: speakSayTTS, stop: stopSayTTS, isAvailable: isSayTTSAvailable } = useSayTTS({
|
||||
enabled: shouldCheckSayAvailability,
|
||||
});
|
||||
const { speak: speakLocalTTS, stop: stopLocalTTS } = useLocalTTS();
|
||||
|
||||
const stop = useCallback(() => {
|
||||
setIsPlaying(false);
|
||||
stopServerTTS();
|
||||
stopSayTTS();
|
||||
stopLocalTTS();
|
||||
browserVoiceService.cancelSpeech();
|
||||
}, [stopServerTTS, stopSayTTS]);
|
||||
}, [stopServerTTS, stopSayTTS, stopLocalTTS]);
|
||||
|
||||
const play = useCallback(async (text: string) => {
|
||||
if (!text.trim()) return;
|
||||
@@ -84,6 +88,13 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else if (voiceProvider === 'local') {
|
||||
await speakLocalTTS(sanitizedText, {
|
||||
speakerId: localTtsVoiceId,
|
||||
speed: speechRate,
|
||||
onEnd: () => setIsPlaying(false),
|
||||
onError: () => setIsPlaying(false),
|
||||
});
|
||||
} else if (voiceProvider === 'say' && isSayTTSAvailable) {
|
||||
const wordsPerMinute = Math.round(100 + (speechRate - 0.5) * 200);
|
||||
await speakSayTTS(sanitizedText, {
|
||||
@@ -129,6 +140,8 @@ export function useMessageTTS(): UseMessageTTSReturn {
|
||||
ttsInputMode,
|
||||
speakServerTTS,
|
||||
speakSayTTS,
|
||||
speakLocalTTS,
|
||||
localTtsVoiceId,
|
||||
stop,
|
||||
]);
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useSessionUIStore } from '@/sync/session-ui-store';
|
||||
import { useSessionPermissions, useSessionTextMessages } from '@/sync/sync-context';
|
||||
import { voiceHooks, isVoiceSessionStarted } from '@/lib/voice';
|
||||
|
||||
/**
|
||||
* Hook that syncs session events (messages, permissions) to the voice agent.
|
||||
* Call this inside VoiceProvider to enable session awareness during voice.
|
||||
*/
|
||||
export function useVoiceContext() {
|
||||
const currentSessionId = useSessionUIStore((s) => s.currentSessionId);
|
||||
const messages = useSessionTextMessages(currentSessionId ?? '');
|
||||
const permissions = useSessionPermissions(currentSessionId ?? '');
|
||||
|
||||
// Track last seen message count to only forward new messages
|
||||
const lastMessageCountRef = useRef(0);
|
||||
|
||||
// Forward new messages to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !messages || messages.length === 0 || !isVoiceSessionStarted()) return;
|
||||
|
||||
const currentCount = messages.length;
|
||||
if (currentCount <= lastMessageCountRef.current) return;
|
||||
|
||||
// Get only new messages (messages since last check)
|
||||
const newMessages = messages.slice(lastMessageCountRef.current);
|
||||
lastMessageCountRef.current = currentCount;
|
||||
|
||||
const formattedMessages = newMessages.map(m => ({
|
||||
role: m.role ?? '',
|
||||
content: m.text,
|
||||
}));
|
||||
|
||||
voiceHooks.onMessages(currentSessionId, formattedMessages);
|
||||
}, [currentSessionId, messages]);
|
||||
|
||||
// Forward permission requests to voice agent
|
||||
useEffect(() => {
|
||||
if (!currentSessionId || !permissions || permissions.length === 0) return;
|
||||
if (!isVoiceSessionStarted()) return;
|
||||
|
||||
const request = permissions[0];
|
||||
if (!request) return;
|
||||
|
||||
voiceHooks.onPermissionRequested(
|
||||
currentSessionId,
|
||||
request.id,
|
||||
request.permission,
|
||||
request.metadata
|
||||
);
|
||||
}, [currentSessionId, permissions]);
|
||||
|
||||
// Reset message count when session changes
|
||||
useEffect(() => {
|
||||
lastMessageCountRef.current = 0;
|
||||
}, [currentSessionId]);
|
||||
}
|
||||
Reference in New Issue
Block a user