Files
openchamber/packages/ui/src/hooks/useDictation.ts
T
Bohdan Triapitsyn de1b85ac56 feat(voice): first-class voice input and local TTS across web, desktop, and mobile (#2018)
Complete rebuild of voice input on a server-authoritative streaming
architecture, replacing the legacy Web Speech / whole-blob / WASM engines
and the dead voice-agent layer (~4k lines removed).

Speech-to-text (dictation):
- Client streams 16 kHz mono PCM16 chunks over /api/dictation/ws with
  seq/ack ordering; buffered audio is retained and replayed on reconnect
- Server transcribes and streams live partial transcripts back;
  segments auto-commit every ~15s with silence suppression and adaptive
  finalization timeouts
- Local provider (default, zero config): sherpa-onnx models in a forked
  worker process — auto-download with progress, staged extraction with
  verification, corrupt-model auto-recovery, idle shutdown after 5 min
- Model catalog with settings picker (accuracy/speed ratings, sizes,
  download/delete): Parakeet TDT v2 (English) and v3 (25 European
  languages, auto-detected), Whisper base and tiny (multilingual, light)
- OpenAI-compatible provider for any Whisper endpoint
- Composer overlay with live transcript, volume meter, timer, and
  cancel / insert / insert-and-send actions; failed transcriptions keep
  their audio for retry or accepting the partial text as-is
- Configurable keyboard shortcut (default mod+alt+v) toggles dictation;
  Enter confirms and Escape cancels while recording
- Overlay is pixel-aligned with the composer (measured footer height,
  matching paddings/typography/gaps) — no layout shift when toggling

Text-to-speech:
- Local Kokoro provider (English, 11 voices) synthesized in the same
  worker via /api/dictation/tts/speak, managed by the shared model
  pipeline; sentence-pipelined playback keeps time-to-first-audio at
  ~1 sentence regardless of message length, and stop cancels in-flight
  synthesis
- Sanitizer keeps inline-code content (strips backticks only), reads
  interword slashes aloud, and removes only absolute file paths

Settings:
- Voice page unified: a single read-aloud toggle owns all playback
  options (the confusing "Enable Voice Mode" is gone); a new "Enable
  voice input" toggle (default on, persisted to settings.json) hides
  the composer mic entirely when disabled

Mobile and transport:
- iOS/Android microphone permissions added (dictation was previously
  impossible on mobile)
- Fixed Android WebSocket upgrades: the Capacitor WebView origin
  (https://localhost) was missing from the packaged-client allowlist,
  403-ing every WS connection — root cause of the old mobile SSE lock,
  which is now removed for all transports

Security and conventions:
- All HTTP routes sit behind the global /api auth gate; the WS upgrade
  explicitly validates the UI session and origin, with oc_url_token
  narrowly allowlisted and covered by tests; the dictation socket mints
  a fresh URL token before connecting
- Routes register before the generic OpenCode proxy; the client goes
  through runtimeFetch/getRuntimeUrlResolver, and runtime switches
  reset the dictation socket
- VS Code deliberately reports dictation as unavailable (no server
  process in that runtime)

CI: workflow Node bumped 20 -> 22 to match the repo engines and fix
better-sqlite3 installs broken by node-gyp@latest on Node 20.

New dependency: sherpa-onnx-node (prebuilt N-API; macOS/Linux x64+arm64,
Windows x64 — Windows-on-ARM falls back to the OpenAI-compatible provider)
2026-07-04 02:48:07 +03:00

409 lines
13 KiB
TypeScript

/**
* Streaming dictation state machine.
*
* Status flow: idle -> recording -> uploading -> idle | failed.
* While recording, mic PCM chunks stream to the server, which sends back live
* partial transcripts. Confirm finalizes and resolves the full text; failed
* dictations retain their audio segments so retry can replay them.
*/
import { useCallback, useEffect, useRef, useState } from 'react';
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
import { useConfigStore } from '@/stores/useConfigStore';
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
export interface UseDictationOptions {
onTranscript?: (text: string) => void;
onError?: (error: Error) => void;
canStart?: () => boolean;
}
export interface UseDictationResult {
status: DictationStatus;
isRecording: boolean;
isProcessing: boolean;
partialTranscript: string;
volume: number;
duration: number;
error: string | null;
errorReason: string | null;
startDictation: () => Promise<void>;
confirmDictation: () => Promise<string | null>;
cancelDictation: () => Promise<void>;
retryFailedDictation: () => Promise<string | null>;
acceptPartialTranscript: () => string | null;
discardFailedDictation: () => void;
}
const DURATION_TICK_MS = 1000;
const toError = (value: unknown): Error =>
value instanceof Error ? value : new Error(String(value));
const getDictationStartOptions = (): DictationStartOptions => {
const state = useConfigStore.getState();
const language = state.sttLanguage?.trim();
if (state.sttProvider === 'openai-compatible') {
return {
provider: 'openai-compatible',
...(language ? { language } : {}),
openaiCompatible: {
baseUrl: state.sttServerUrl,
model: state.sttModel,
...(state.sttApiKey ? { apiKey: state.sttApiKey } : {}),
},
};
}
return {
provider: 'local',
...(language ? { language } : {}),
localModel: state.sttLocalModel,
};
};
export function useDictation(options: UseDictationOptions = {}): UseDictationResult {
const { onTranscript, onError, canStart } = options;
const [status, setStatus] = useState<DictationStatus>('idle');
const [partialTranscript, setPartialTranscript] = useState('');
const [duration, setDuration] = useState(0);
const [error, setError] = useState<string | null>(null);
const [errorReason, setErrorReason] = useState<string | null>(null);
const statusRef = useRef(status);
useEffect(() => {
statusRef.current = status;
}, [status]);
const latestPartialRef = useRef('');
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const actionGateRef = useRef({ starting: false, confirming: false, cancelling: false });
const onTranscriptRef = useRef(onTranscript);
const onErrorRef = useRef(onError);
useEffect(() => {
onTranscriptRef.current = onTranscript;
onErrorRef.current = onError;
}, [onTranscript, onError]);
const senderRef = useRef<DictationStreamSender | null>(null);
if (!senderRef.current) {
senderRef.current = new DictationStreamSender({
client: dictationClient,
getStartOptions: getDictationStartOptions,
});
}
const stopDurationTracking = useCallback(() => {
if (durationIntervalRef.current) {
clearInterval(durationIntervalRef.current);
durationIntervalRef.current = null;
}
}, []);
const startDurationTracking = useCallback(() => {
if (durationIntervalRef.current) {
return;
}
durationIntervalRef.current = setInterval(() => {
setDuration((prev) => prev + 1);
}, DURATION_TICK_MS);
}, []);
const reportError = useCallback((err: unknown) => {
const normalized = toError(err);
setError(normalized.message);
const reason = (normalized as Error & { reasonCode?: string }).reasonCode;
setErrorReason(typeof reason === 'string' ? reason : null);
onErrorRef.current?.(normalized);
}, []);
const clearError = useCallback(() => {
setError(null);
setErrorReason(null);
}, []);
const clearStreamingState = useCallback(() => {
senderRef.current?.clearAll();
latestPartialRef.current = '';
setPartialTranscript('');
}, []);
// Live partial transcripts for the active dictation.
useEffect(() => {
return dictationClient.onPartial((dictationId, text) => {
const activeDictationId = senderRef.current?.getDictationId();
if (!activeDictationId || dictationId !== activeDictationId) {
return;
}
latestPartialRef.current = text;
setPartialTranscript(text);
});
}, []);
// Restart the stream (replaying buffered segments) after a reconnect.
useEffect(() => {
return dictationClient.subscribeConnectionStatus((connected) => {
if (!connected) {
return;
}
if (statusRef.current !== 'recording') {
return;
}
void senderRef.current?.restartStream().catch((err) => {
reportError(err);
});
});
}, [reportError]);
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
senderRef.current?.enqueueSegment(audioData);
},
onError: (err) => {
onErrorRef.current?.(err);
},
});
const audioStopRef = useRef(audio.stop);
useEffect(() => {
audioStopRef.current = audio.stop;
}, [audio.stop]);
const handleSuccess = useCallback(
(text: string): string | null => {
setDuration(0);
setStatus('idle');
const transcriptText = text.trim().length > 0 ? text.trim() : latestPartialRef.current.trim();
clearStreamingState();
if (!transcriptText) {
return null;
}
onTranscriptRef.current?.(transcriptText);
return transcriptText;
},
[clearStreamingState],
);
const handleFailure = useCallback(
(failure: unknown) => {
if (senderRef.current?.hasSegments()) {
setStatus('failed');
} else {
setStatus('idle');
}
reportError(failure);
},
[reportError],
);
const startDictation = useCallback(async () => {
const gate = actionGateRef.current;
if (gate.starting || gate.confirming || gate.cancelling) {
return;
}
if (statusRef.current !== 'idle') {
return;
}
if (canStart && !canStart()) {
return;
}
gate.starting = true;
clearError();
setPartialTranscript('');
setDuration(0);
setStatus('recording');
statusRef.current = 'recording';
clearStreamingState();
try {
await audio.start();
startDurationTracking();
// Open the stream eagerly so partials start flowing immediately.
await senderRef.current?.restartStream().catch((err) => {
// Non-fatal: segments buffer locally and finish() retries the
// start, but surface the reason (e.g. model downloading) so
// the overlay can show it.
reportError(err);
});
} catch (err) {
await audio.stop().catch(() => undefined);
stopDurationTracking();
setStatus('idle');
statusRef.current = 'idle';
reportError(err);
} finally {
gate.starting = false;
}
}, [audio, canStart, clearError, clearStreamingState, reportError, startDurationTracking, stopDurationTracking]);
const cancelDictation = useCallback(async () => {
const gate = actionGateRef.current;
if (gate.cancelling) {
return;
}
if (statusRef.current !== 'recording' && statusRef.current !== 'uploading') {
return;
}
gate.cancelling = true;
stopDurationTracking();
setDuration(0);
clearError();
// Optimistic: dismiss the overlay immediately. Tearing down the audio
// graph (AudioContext.close) can take up to ~1s on mobile WebViews and
// must not delay the visible response to Cancel.
setStatus('idle');
statusRef.current = 'idle';
try {
senderRef.current?.cancel();
} catch {
// no-op
}
clearStreamingState();
try {
await audio.stop();
} catch {
// Cancelled anyway; mic teardown failures are not user-actionable.
} finally {
gate.cancelling = false;
}
}, [audio, clearError, clearStreamingState, stopDurationTracking]);
const confirmDictation = useCallback(async (): Promise<string | null> => {
const gate = actionGateRef.current;
if (gate.confirming) {
return null;
}
if (statusRef.current !== 'recording') {
return null;
}
gate.confirming = true;
clearError();
stopDurationTracking();
try {
await audio.stop();
setStatus('uploading');
statusRef.current = 'uploading';
const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
if (finalSeq < 0) {
return handleSuccess('');
}
const result = await senderRef.current!.finish(finalSeq);
return handleSuccess(result.text);
} catch (err) {
handleFailure(err);
return null;
} finally {
gate.confirming = false;
}
}, [audio, clearError, handleFailure, handleSuccess, stopDurationTracking]);
const retryFailedDictation = useCallback(async (): Promise<string | null> => {
if (statusRef.current !== 'failed' || !senderRef.current?.hasSegments()) {
return null;
}
clearError();
setStatus('uploading');
statusRef.current = 'uploading';
try {
senderRef.current.resetStreamForReplay();
const finalSeq = senderRef.current.getFinalSeq();
const result = await senderRef.current.finish(finalSeq);
return handleSuccess(result.text);
} catch (err) {
handleFailure(err);
return null;
}
}, [clearError, handleFailure, handleSuccess]);
/**
* Failed dictations still hold the last streamed partial transcript.
* Accept it as-is instead of retrying the full transcription.
*/
const acceptPartialTranscript = useCallback((): string | null => {
if (statusRef.current !== 'failed') {
return null;
}
const text = latestPartialRef.current.trim();
setDuration(0);
setStatus('idle');
statusRef.current = 'idle';
clearError();
try {
senderRef.current?.cancel();
} catch {
// no-op
}
clearStreamingState();
if (!text) {
return null;
}
onTranscriptRef.current?.(text);
return text;
}, [clearError, clearStreamingState]);
const discardFailedDictation = useCallback(() => {
setDuration(0);
setStatus('idle');
statusRef.current = 'idle';
clearError();
clearStreamingState();
}, [clearError, clearStreamingState]);
// While recording without an open stream (e.g. the model is still
// downloading), retry the stream start so live partials kick in as soon
// as the provider becomes ready. Buffered segments replay on success.
useEffect(() => {
if (status !== 'recording' || errorReason !== 'model_download_in_progress') {
return;
}
const interval = setInterval(() => {
if (statusRef.current !== 'recording' || senderRef.current?.getDictationId()) {
return;
}
senderRef.current?.restartStream().then(() => {
clearError();
}).catch((err) => {
reportError(err);
});
}, 3000);
return () => clearInterval(interval);
}, [status, errorReason, clearError, reportError]);
useEffect(() => {
return () => {
stopDurationTracking();
void audioStopRef.current().catch(() => undefined);
senderRef.current?.cancel();
};
}, [stopDurationTracking]);
return {
status,
isRecording: status === 'recording',
isProcessing: status === 'uploading',
partialTranscript,
volume: audio.volume,
duration,
error,
errorReason,
startDictation,
confirmDictation,
cancelDictation,
retryFailedDictation,
acceptPartialTranscript,
discardFailedDictation,
};
}