feat(dictation): transcribe after recording instead of live
Parakeet is an offline model trained on whole utterances, so re-decoding the growing buffer to animate a live transcript cost O(n^2) work for a result the final decode replaced. Sessions now decode once per committed segment, and the composer shows a scrolling waveform of the mic level instead of running text. Long dictations split at a pause once past 60s (hard cap 90s) instead of on a blind 15s timer, so cuts no longer land mid-word. Committed segments decode while the user is still speaking: a 185s dictation returns 4.1s after stop instead of 11.0s, with identical text (816 vs 817 words). Also fixes two ways the stream manager could silently drop transcribed audio. It now counts the commits it issued instead of trusting the session's echoed events, so a commit still in flight when the client finishes can no longer be left out of the final text. And segment byte/peak accounting is reset where the commit is issued rather than when the event arrives, which could mistake the tail of a dictation for silence and clear it.
This commit is contained in:
Bohdan Triapitsyn
@@ -5,6 +5,10 @@
|
||||
* area uses the same paddings/typography as the textarea and the action row
|
||||
* reuses the footer icon-button styling — so toggling dictation causes no
|
||||
* vertical shift.
|
||||
*
|
||||
* No text appears while recording. The server transcribes the audio once the
|
||||
* user stops, so the overlay shows the recording state and then Transcribing.
|
||||
* The only transcript rendered here is the salvage text of a failed dictation.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
@@ -15,6 +19,7 @@ import { useThemeSystem } from '@/contexts/useThemeSystem';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
import { useDictation } from '@/hooks/useDictation';
|
||||
import { DictationWaveform } from '@/components/dictation/DictationWaveform';
|
||||
import { isDictationCaptureSupported } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { isVSCodeRuntime } from '@/lib/desktop';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
@@ -50,25 +55,6 @@ const formatDuration = (seconds: number): string => {
|
||||
return `${mins}:${String(secs).padStart(2, '0')}`;
|
||||
};
|
||||
|
||||
const VolumeMeter: React.FC<{ volume: number }> = ({ volume }) => {
|
||||
const { currentTheme } = useThemeSystem();
|
||||
return (
|
||||
<div
|
||||
className="h-1.5 w-16 flex-shrink-0 overflow-hidden rounded-full"
|
||||
style={{ backgroundColor: currentTheme.colors.interactive.border }}
|
||||
aria-hidden="true"
|
||||
>
|
||||
<div
|
||||
className="h-full rounded-full transition-[width] duration-75"
|
||||
style={{
|
||||
width: `${Math.round(Math.min(1, volume) * 100)}%`,
|
||||
backgroundColor: currentTheme.colors.primary.base,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Polls the dictation status route while the local model is downloading and
|
||||
* returns the download percent (null while unknown / not downloading).
|
||||
@@ -162,7 +148,7 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
const {
|
||||
status,
|
||||
partialTranscript,
|
||||
volume,
|
||||
subscribeLevel,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
@@ -399,7 +385,7 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
{/* Measured for the composer-growth report — keep all
|
||||
transcript/placeholder/error content inside. */}
|
||||
<div ref={transcriptContentRef}>
|
||||
{partialTranscript ? (
|
||||
{status === 'failed' && partialTranscript ? (
|
||||
<p className="typography-markdown md:typography-ui-label whitespace-pre-wrap" style={{ color: currentTheme.colors.surface.foreground }}>
|
||||
{partialTranscript}
|
||||
</p>
|
||||
@@ -436,8 +422,8 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
style={{ backgroundColor: currentTheme.colors.status.error }}
|
||||
/>
|
||||
</span>
|
||||
<VolumeMeter volume={volume} />
|
||||
<span className="typography-meta tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||
<DictationWaveform subscribeLevel={subscribeLevel} className="block h-4 min-w-0 flex-1" />
|
||||
<span className="typography-meta flex-shrink-0 tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||
{formatDuration(duration)}
|
||||
</span>
|
||||
</>
|
||||
@@ -446,7 +432,7 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
) : null}
|
||||
{/* Same inter-control gap as the composer's right cluster:
|
||||
gap-x-1 on mobile, md:gap-x-3 on desktop. */}
|
||||
<div className={cn('ml-auto flex items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
|
||||
<div className={cn('ml-auto flex flex-shrink-0 items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
|
||||
{status === 'recording' ? (
|
||||
<>
|
||||
<button
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Scrolling microphone level history for the dictation overlay.
|
||||
*
|
||||
* Newest sample is at the right edge and the history scrolls left, so the row
|
||||
* reads as a live recording trace rather than a single level bar. Silence
|
||||
* renders as a dot, speech as a rounded bar.
|
||||
*
|
||||
* Drawn on a canvas fed by a level subscription: the level updates ~12 times a
|
||||
* second, and routing that through React state would re-render the whole
|
||||
* dictation overlay at the same rate.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
|
||||
import { useThemeSystem } from '@/contexts/useThemeSystem';
|
||||
import type { DictationLevelListener } from '@/lib/dictation/use-dictation-audio-source';
|
||||
|
||||
interface DictationWaveformProps {
|
||||
subscribeLevel: (listener: DictationLevelListener) => () => void;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const BAR_WIDTH = 2;
|
||||
const BAR_GAP = 3;
|
||||
const BAR_PITCH = BAR_WIDTH + BAR_GAP;
|
||||
/** One sample per bar; ~16 bars/s scrolls at a readable speed. */
|
||||
const SAMPLE_INTERVAL_MS = 60;
|
||||
/** Raise quiet speech so normal talking uses most of the height. */
|
||||
const LEVEL_CURVE = 0.65;
|
||||
|
||||
export const DictationWaveform: React.FC<DictationWaveformProps> = ({ subscribeLevel, className }) => {
|
||||
const { currentTheme } = useThemeSystem();
|
||||
const canvasRef = React.useRef<HTMLCanvasElement | null>(null);
|
||||
const barColor = currentTheme.colors.surface.mutedForeground;
|
||||
|
||||
React.useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) {
|
||||
return;
|
||||
}
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Peak-hold between samples: a short loud syllable must not be missed
|
||||
// just because it landed between two frames.
|
||||
let peakSinceSample = 0;
|
||||
const unsubscribe = subscribeLevel((level) => {
|
||||
peakSinceSample = Math.max(peakSinceSample, level);
|
||||
});
|
||||
|
||||
const bars: number[] = [];
|
||||
let cssWidth = 0;
|
||||
let cssHeight = 0;
|
||||
|
||||
const resize = () => {
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
cssWidth = rect.width;
|
||||
cssHeight = rect.height;
|
||||
canvas.width = Math.max(1, Math.round(cssWidth * dpr));
|
||||
canvas.height = Math.max(1, Math.round(cssHeight * dpr));
|
||||
context.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
};
|
||||
resize();
|
||||
const observer = new ResizeObserver(resize);
|
||||
observer.observe(canvas);
|
||||
|
||||
const draw = () => {
|
||||
const capacity = Math.max(1, Math.floor(cssWidth / BAR_PITCH));
|
||||
while (bars.length > capacity) {
|
||||
bars.shift();
|
||||
}
|
||||
|
||||
context.clearRect(0, 0, cssWidth, cssHeight);
|
||||
context.strokeStyle = barColor;
|
||||
context.fillStyle = barColor;
|
||||
context.lineWidth = BAR_WIDTH;
|
||||
context.lineCap = 'round';
|
||||
|
||||
const centerY = cssHeight / 2;
|
||||
const maxHeight = Math.max(BAR_WIDTH, cssHeight);
|
||||
// Anchor the newest bar to the right edge; older bars trail left.
|
||||
const rightX = cssWidth - BAR_WIDTH / 2;
|
||||
|
||||
for (let i = 0; i < bars.length; i++) {
|
||||
const x = rightX - (bars.length - 1 - i) * BAR_PITCH;
|
||||
if (x < BAR_WIDTH / 2) {
|
||||
continue;
|
||||
}
|
||||
const height = BAR_WIDTH + (maxHeight - BAR_WIDTH) * Math.pow(bars[i], LEVEL_CURVE);
|
||||
// Round caps add BAR_WIDTH/2 past each end of the stroke, so the
|
||||
// stroke itself is the height minus one cap diameter.
|
||||
const half = (height - BAR_WIDTH) / 2;
|
||||
context.beginPath();
|
||||
if (half < 0.25) {
|
||||
context.arc(x, centerY, BAR_WIDTH / 2, 0, Math.PI * 2);
|
||||
context.fill();
|
||||
} else {
|
||||
context.moveTo(x, centerY - half);
|
||||
context.lineTo(x, centerY + half);
|
||||
context.stroke();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let frame = 0;
|
||||
let lastSampleAt = 0;
|
||||
const tick = (now: number) => {
|
||||
frame = requestAnimationFrame(tick);
|
||||
if (now - lastSampleAt < SAMPLE_INTERVAL_MS) {
|
||||
return;
|
||||
}
|
||||
lastSampleAt = now;
|
||||
bars.push(peakSinceSample);
|
||||
peakSinceSample = 0;
|
||||
draw();
|
||||
};
|
||||
frame = requestAnimationFrame(tick);
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
observer.disconnect();
|
||||
unsubscribe();
|
||||
};
|
||||
}, [subscribeLevel, barColor]);
|
||||
|
||||
return <canvas ref={canvasRef} className={className} aria-hidden="true" />;
|
||||
};
|
||||
@@ -2,16 +2,19 @@
|
||||
* Streaming dictation state machine.
|
||||
*
|
||||
* Status flow: idle -> recording -> uploading -> idle | failed.
|
||||
* While recording, mic PCM chunks stream to the server, which sends back live
|
||||
* partial transcripts. Confirm finalizes and resolves the full text; failed
|
||||
* dictations retain their audio segments so retry can replay them.
|
||||
* While recording, mic PCM chunks stream to the server, which transcribes them
|
||||
* segment by segment; confirm finalizes and resolves the full text. Nothing is
|
||||
* shown while recording — `partialTranscript` holds whatever the server has
|
||||
* transcribed so far and exists only so a failed dictation can be salvaged
|
||||
* instead of losing minutes of speech. Failed dictations also retain their
|
||||
* audio segments so retry can replay them.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
|
||||
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
|
||||
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { useDictationAudioSource, type DictationLevelListener } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
|
||||
@@ -26,8 +29,10 @@ export interface UseDictationResult {
|
||||
status: DictationStatus;
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
/** Server-side transcript so far; recovery only, never shown while recording. */
|
||||
partialTranscript: string;
|
||||
volume: number;
|
||||
/** Subscribe to the normalized (0..1) mic level for the waveform. */
|
||||
subscribeLevel: (listener: DictationLevelListener) => () => void;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
errorReason: string | null;
|
||||
@@ -133,7 +138,8 @@ export function useDictation(options: UseDictationOptions = {}): UseDictationRes
|
||||
setPartialTranscript('');
|
||||
}, []);
|
||||
|
||||
// Live partial transcripts for the active dictation.
|
||||
// Transcripts of segments the server has already committed. Not rendered
|
||||
// while recording; kept so a failed dictation can be salvaged.
|
||||
useEffect(() => {
|
||||
return dictationClient.onPartial((dictationId, text) => {
|
||||
const activeDictationId = senderRef.current?.getDictationId();
|
||||
@@ -223,7 +229,8 @@ export function useDictation(options: UseDictationOptions = {}): UseDictationRes
|
||||
try {
|
||||
await audio.start();
|
||||
startDurationTracking();
|
||||
// Open the stream eagerly so partials start flowing immediately.
|
||||
// Open the stream eagerly so audio uploads while the user speaks
|
||||
// and only the tail is left to transcribe on stop.
|
||||
await senderRef.current?.restartStream().catch((err) => {
|
||||
// Non-fatal: segments buffer locally and finish() retries the
|
||||
// start, but surface the reason (e.g. model downloading) so
|
||||
@@ -394,7 +401,7 @@ export function useDictation(options: UseDictationOptions = {}): UseDictationRes
|
||||
isRecording: status === 'recording',
|
||||
isProcessing: status === 'uploading',
|
||||
partialTranscript,
|
||||
volume: audio.volume,
|
||||
subscribeLevel: audio.subscribeLevel,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
|
||||
@@ -4,20 +4,27 @@
|
||||
* Captures mono audio via getUserMedia, taps it with a ScriptProcessorNode
|
||||
* (universally supported, including iOS WKWebView), resamples Float32 to
|
||||
* 16 kHz PCM16LE, and emits ~1-second base64 chunks plus a normalized RMS
|
||||
* volume for the level meter.
|
||||
* level for the waveform.
|
||||
*
|
||||
* The level is delivered by subscription rather than React state: it updates
|
||||
* on every audio callback (~12 Hz), and routing that through state re-rendered
|
||||
* the whole dictation overlay at the same rate.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useRef } from 'react';
|
||||
|
||||
export interface DictationAudioSourceConfig {
|
||||
onPcmSegment: (base64Pcm: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
export type DictationLevelListener = (level: number) => void;
|
||||
|
||||
export interface DictationAudioSource {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
volume: number;
|
||||
/** Subscribe to the normalized (0..1) mic level. Returns an unsubscribe. */
|
||||
subscribeLevel: (listener: DictationLevelListener) => () => void;
|
||||
}
|
||||
|
||||
const OUTPUT_RATE = 16000;
|
||||
@@ -125,7 +132,18 @@ export const isDictationCaptureSupported = (): boolean => {
|
||||
};
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const [volume, setVolume] = useState(0);
|
||||
const levelListenersRef = useRef(new Set<DictationLevelListener>());
|
||||
const emitLevel = useCallback((level: number) => {
|
||||
for (const listener of levelListenersRef.current) {
|
||||
listener(level);
|
||||
}
|
||||
}, []);
|
||||
const subscribeLevel = useCallback((listener: DictationLevelListener) => {
|
||||
levelListenersRef.current.add(listener);
|
||||
return () => {
|
||||
levelListenersRef.current.delete(listener);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
@@ -196,7 +214,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
sumSquares += input[i] * input[i];
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
setVolume(Math.min(1, Math.max(0, rms * 2)));
|
||||
emitLevel(Math.min(1, Math.max(0, rms * 2)));
|
||||
|
||||
const next = resampleToPcm16(input, context.sampleRate, OUTPUT_RATE);
|
||||
graph.pending = concatInt16(graph.pending, next);
|
||||
@@ -227,12 +245,12 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
graphRef.current = emptyGraph();
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
}, []);
|
||||
}, [emitLevel]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
const graph = graphRef.current;
|
||||
graph.started = false;
|
||||
setVolume(0);
|
||||
emitLevel(0);
|
||||
|
||||
if (graph.processor) {
|
||||
try {
|
||||
@@ -272,7 +290,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
if (graphRef.current === graph) {
|
||||
graphRef.current = emptyGraph();
|
||||
}
|
||||
}, []);
|
||||
}, [emitLevel]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
@@ -294,8 +312,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
}
|
||||
},
|
||||
stop,
|
||||
volume,
|
||||
subscribeLevel,
|
||||
}),
|
||||
[start, stop, volume],
|
||||
[start, stop, subscribeLevel],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
# Dictation module
|
||||
|
||||
Server-authoritative streaming speech-to-text for the chat composer, plus
|
||||
local text-to-speech. The client streams 16 kHz mono PCM16 chunks (base64)
|
||||
over a WebSocket; the server runs the transcription and streams live partial
|
||||
transcripts back.
|
||||
Server-authoritative speech-to-text for the chat composer, plus local
|
||||
text-to-speech. The client streams 16 kHz mono PCM16 chunks (base64) over a
|
||||
WebSocket while the user speaks; the server buffers them and transcribes each
|
||||
segment exactly once, when the segment is committed.
|
||||
|
||||
Transcription is deliberately not incremental. Parakeet is an offline model
|
||||
trained on whole utterances, so re-decoding the growing buffer to animate a
|
||||
live transcript costs O(n^2) work for a result the final decode replaces. The
|
||||
composer shows no text while recording and inserts the full transcript on
|
||||
stop.
|
||||
|
||||
Local TTS (Kokoro via sherpa-onnx OfflineTts) runs in the same worker process
|
||||
and is exposed as `POST /api/dictation/tts/speak` (JSON `{text, speakerId?,
|
||||
@@ -21,9 +27,9 @@ same status/download/delete routes.
|
||||
Created from the startup pipeline (`startup-pipeline-runtime.js`) before
|
||||
the generic OpenCode proxy so routes are not shadowed.
|
||||
- `stream-manager.js` — `DictationStreamManager`, one per WS connection.
|
||||
Chunk reordering by `seq` + ack, resampling to the provider rate,
|
||||
auto-commit every ~15 s of audio, silence suppression by PCM peak,
|
||||
partial-transcript concatenation, adaptive finalization timeout.
|
||||
Chunk reordering by `seq` + ack, resampling to the provider rate, segment
|
||||
splitting, silence suppression by PCM peak, partial-transcript
|
||||
concatenation, adaptive finalization timeout.
|
||||
- `service.js` — provider resolution and readiness. Providers:
|
||||
- `local` (default): sherpa-onnx Parakeet TDT in a forked worker process.
|
||||
Models auto-download in the background on first use; while missing, the
|
||||
@@ -33,7 +39,7 @@ same status/download/delete routes.
|
||||
OpenAI-compatible `/v1/audio/transcriptions` endpoint
|
||||
(`openai-compatible-session.js`, reuses `../tts/stt.js`).
|
||||
- `local/` — worker process + client (IPC, idle shutdown TTL), sherpa
|
||||
recognizer engine and realtime session (throttled re-decode for partials),
|
||||
recognizer engine and segment session (one decode per committed segment),
|
||||
model catalog and downloader. The native `sherpa-onnx-node` addon is only
|
||||
ever loaded inside the worker process.
|
||||
- `audio.js` — PCM16 helpers: format parsing, peak, WAV wrapping, streaming
|
||||
@@ -53,9 +59,27 @@ Server → client: `ready`, `ack {ackSeq}`, `partial {text}`,
|
||||
`{ provider: 'local' | 'openai-compatible', language?, localModel?,
|
||||
openaiCompatible?: { baseUrl, model, apiKey } }`.
|
||||
|
||||
## Segmentation
|
||||
|
||||
A dictation is one segment unless it runs long. Past `segmentMinSeconds`
|
||||
(60 s) the manager commits on the first silent chunk, so cuts land at a pause
|
||||
rather than mid-word; `segmentMaxSeconds` (90 s) is a hard cap for speech with
|
||||
no pause in it. Client chunks are ~1 s, so "silent chunk" is roughly a second
|
||||
of silence.
|
||||
|
||||
The bounds exist because Parakeet is a full-attention conformer: decode cost
|
||||
and peak memory grow quadratically with segment length. Measured on Parakeet
|
||||
v3 int8 with 2 threads: 60 s took 2.1 s and +90 MB, 180 s took 9.3 s and
|
||||
+490 MB, 300 s took 21.3 s and +1.5 GB. Committed segments decode while the
|
||||
user is still speaking, so only the tail is left to transcribe on stop.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Never load `sherpa-onnx-node` in the main server process.
|
||||
- Transcription happens on commit only; sessions never emit non-final
|
||||
transcripts. The `partial` messages a client receives are the concatenation
|
||||
of already-committed segments, and exist so a dictation that fails partway
|
||||
can be salvaged instead of losing minutes of speech.
|
||||
- The stream manager acks only the highest contiguous seq; the client is
|
||||
expected to retain unacked segments for retry/replay.
|
||||
- Silence-only segments (peak < 300) are cleared, never committed, so
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
/**
|
||||
* Sherpa-onnx offline recognizer engine (NeMo transducer / Parakeet) plus a
|
||||
* realtime streaming transcription session that re-decodes the accumulated
|
||||
* segment audio on a throttle to produce live partial transcripts.
|
||||
* segment transcription session that decodes each segment exactly once, when
|
||||
* the segment is committed.
|
||||
*
|
||||
* Parakeet is an offline model: it is trained to see a whole utterance at
|
||||
* once. Decoding the accumulated audio repeatedly to animate a live transcript
|
||||
* costs O(n^2) work for a result the final decode throws away, so this session
|
||||
* only decodes on commit.
|
||||
*
|
||||
* Runs inside the dictation worker process only — never load the native
|
||||
* addon in the main server process.
|
||||
@@ -147,31 +152,26 @@ export class SherpaOfflineRecognizerEngine {
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming transcription session backed by the offline recognizer.
|
||||
* Accumulates the current segment's PCM and re-decodes it at most every
|
||||
* `minDecodeIntervalMs` to emit non-final partial transcripts; `commit()`
|
||||
* finalizes the segment and starts a new one.
|
||||
* Segment transcription session backed by the offline recognizer.
|
||||
* Accumulates the current segment's PCM and decodes it once in `commit()`,
|
||||
* which emits the segment's final transcript and starts a new segment.
|
||||
*
|
||||
* Implements the StreamingTranscriptionSession contract used by
|
||||
* DictationStreamManager.
|
||||
* DictationStreamManager. It never emits non-final transcripts: the manager's
|
||||
* live `partial` messages are the concatenation of already-committed segments.
|
||||
*/
|
||||
export class SherpaRealtimeTranscriptionSession extends EventEmitter {
|
||||
export class SherpaSegmentTranscriptionSession extends EventEmitter {
|
||||
/**
|
||||
* @param {{ engine: SherpaOfflineRecognizerEngine, minDecodeIntervalMs?: number }} params
|
||||
* @param {{ engine: SherpaOfflineRecognizerEngine }} params
|
||||
*/
|
||||
constructor({ engine, minDecodeIntervalMs }) {
|
||||
constructor({ engine }) {
|
||||
super();
|
||||
this.engine = engine;
|
||||
this.requiredSampleRate = engine.sampleRate;
|
||||
this.minDecodeIntervalMs = minDecodeIntervalMs ?? 350;
|
||||
this.connected = false;
|
||||
this.currentSegmentId = null;
|
||||
this.previousSegmentId = null;
|
||||
this.lastPartialText = '';
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
this.lastDecodeAt = 0;
|
||||
this.decoding = false;
|
||||
this.pendingDecode = false;
|
||||
}
|
||||
|
||||
async connect() {
|
||||
@@ -184,39 +184,38 @@ export class SherpaRealtimeTranscriptionSession extends EventEmitter {
|
||||
|
||||
appendPcm16(chunk) {
|
||||
if (!this.connected || !this.currentSegmentId) {
|
||||
this.emit('error', new Error('Sherpa realtime session not connected'));
|
||||
this.emit('error', new Error('Sherpa transcription session not connected'));
|
||||
return;
|
||||
}
|
||||
this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
|
||||
this.maybeDecode(false).catch((err) => {
|
||||
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
}
|
||||
|
||||
commit() {
|
||||
if (!this.connected || !this.currentSegmentId) {
|
||||
this.emit('error', new Error('Sherpa realtime session not connected'));
|
||||
this.emit('error', new Error('Sherpa transcription session not connected'));
|
||||
return;
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
await this.maybeDecode(true);
|
||||
const finalText = this.lastPartialText;
|
||||
const segmentId = this.currentSegmentId;
|
||||
const previousSegmentId = this.previousSegmentId;
|
||||
const segmentId = this.currentSegmentId;
|
||||
const previousSegmentId = this.previousSegmentId;
|
||||
const pcm16 = this.pcm16;
|
||||
|
||||
this.emit('committed', { segmentId, previousSegmentId });
|
||||
this.emit('transcript', { segmentId, transcript: finalText, isFinal: true });
|
||||
// Start the next segment before decoding: decoding blocks the worker for
|
||||
// seconds on long segments, and audio for the next one keeps arriving.
|
||||
this.previousSegmentId = segmentId;
|
||||
this.currentSegmentId = randomUUID();
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
|
||||
this.previousSegmentId = segmentId;
|
||||
this.currentSegmentId = randomUUID();
|
||||
this.lastPartialText = '';
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
} catch (err) {
|
||||
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
})();
|
||||
this.emit('committed', { segmentId, previousSegmentId });
|
||||
|
||||
let transcript;
|
||||
try {
|
||||
transcript = this.engine.decodePcm16(pcm16);
|
||||
} catch (err) {
|
||||
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
||||
return;
|
||||
}
|
||||
this.emit('transcript', { segmentId, transcript, isFinal: true });
|
||||
}
|
||||
|
||||
clear() {
|
||||
@@ -225,7 +224,6 @@ export class SherpaRealtimeTranscriptionSession extends EventEmitter {
|
||||
}
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
this.currentSegmentId = randomUUID();
|
||||
this.lastPartialText = '';
|
||||
}
|
||||
|
||||
close() {
|
||||
@@ -233,45 +231,4 @@ export class SherpaRealtimeTranscriptionSession extends EventEmitter {
|
||||
this.currentSegmentId = null;
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
}
|
||||
|
||||
async maybeDecode(force) {
|
||||
if (!this.connected || !this.currentSegmentId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.decoding) {
|
||||
this.pendingDecode = true;
|
||||
return;
|
||||
}
|
||||
|
||||
this.decoding = true;
|
||||
try {
|
||||
const decodeStartedAt = Date.now();
|
||||
const text = this.engine.decodePcm16(this.pcm16);
|
||||
this.lastDecodeAt = Date.now();
|
||||
// Adaptive throttle: on slow hardware (or heavy models) re-decoding the
|
||||
// growing segment every 350ms would monopolize the worker. Space partial
|
||||
// decodes to ~1.5x the observed decode time.
|
||||
this.minDecodeIntervalMs = Math.max(350, (this.lastDecodeAt - decodeStartedAt) * 1.5);
|
||||
if (text !== this.lastPartialText) {
|
||||
this.lastPartialText = text;
|
||||
this.emit('transcript', {
|
||||
segmentId: this.currentSegmentId,
|
||||
transcript: text,
|
||||
isFinal: false,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
this.decoding = false;
|
||||
if (this.pendingDecode) {
|
||||
this.pendingDecode = false;
|
||||
await this.maybeDecode(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
import {
|
||||
SherpaOfflineRecognizerEngine,
|
||||
SherpaRealtimeTranscriptionSession,
|
||||
SherpaSegmentTranscriptionSession,
|
||||
} from './sherpa-recognizer.js';
|
||||
import { SherpaTtsEngine } from './sherpa-tts.js';
|
||||
import { getLocalSttModelDir, getLocalSttModelSpec } from './model-catalog.js';
|
||||
@@ -126,7 +126,7 @@ async function handleRequest(message) {
|
||||
case 'session.create': {
|
||||
cleanupSession(message.sessionId);
|
||||
const engine = getEngine(message.modelsDir, message.modelId);
|
||||
const session = new SherpaRealtimeTranscriptionSession({ engine });
|
||||
const session = new SherpaSegmentTranscriptionSession({ engine });
|
||||
session.on('committed', (payload) => {
|
||||
sendToParent({ type: 'session.committed', sessionId: message.sessionId, payload });
|
||||
});
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
* endpoints (faster-whisper, whisper.cpp, OpenAI, ...).
|
||||
*
|
||||
* The Whisper HTTP API cannot stream, so audio is buffered per segment and
|
||||
* transcribed on commit(). Live partials therefore only advance at segment
|
||||
* boundaries (the DictationStreamManager auto-commits every ~15s of speech).
|
||||
* transcribed on commit(). This matches how the local session behaves: the
|
||||
* DictationStreamManager splits long dictations at pauses, and everything
|
||||
* shorter is one request on stop.
|
||||
*
|
||||
* Implements the StreamingTranscriptionSession contract used by
|
||||
* DictationStreamManager.
|
||||
|
||||
@@ -7,23 +7,54 @@
|
||||
* Responsibilities:
|
||||
* - Reorders inbound chunks by `seq` and acks the highest contiguous seq.
|
||||
* - Resamples client PCM (16 kHz by default) to the provider's required rate.
|
||||
* - Auto-commits a segment every `autoCommitSeconds` of audio, but clears
|
||||
* silence-only segments instead of committing them.
|
||||
* - Segments long dictations at natural pauses: past `segmentMinSeconds` of
|
||||
* audio it commits on the first silent chunk, and `segmentMaxSeconds` is a
|
||||
* hard cap for speech with no pause in it. Silence-only segments are
|
||||
* cleared instead of committed.
|
||||
* - Concatenates per-segment transcripts into live partials and emits the
|
||||
* final text once every committed segment has a final transcript.
|
||||
* final text once every committed segment has a final transcript. The
|
||||
* manager counts the commits it issued rather than trusting the session's
|
||||
* echoed events, so a commit still in flight when the client finishes
|
||||
* cannot be silently dropped from the transcript.
|
||||
* - Applies an adaptive finalization timeout budget based on pending work.
|
||||
*/
|
||||
|
||||
import { Pcm16MonoResampler, parsePcmRateFromFormat, pcm16lePeakAbs } from './audio.js';
|
||||
|
||||
const DEFAULT_FINAL_TIMEOUT_MS = 10000;
|
||||
const DEFAULT_AUTO_COMMIT_SECONDS = 15;
|
||||
// Parakeet is a full-attention conformer: decode cost and peak memory grow
|
||||
// quadratically with segment length (measured: 60s -> 2.1s/+90MB,
|
||||
// 300s -> 21.3s/+1.5GB). Segmenting keeps a long dictation off that curve and
|
||||
// lets committed segments decode while the user is still speaking, so only the
|
||||
// tail is left to transcribe on stop. Typical dictations are shorter than the
|
||||
// minimum and are decoded as a single segment.
|
||||
const DEFAULT_SEGMENT_MIN_SECONDS = 60;
|
||||
const DEFAULT_SEGMENT_MAX_SECONDS = 90;
|
||||
const FINAL_TIMEOUT_MAX_MS = 5 * 60 * 1000;
|
||||
const FINAL_TIMEOUT_PER_PENDING_SEGMENT_MS = 15 * 1000;
|
||||
const FINAL_TIMEOUT_PER_PENDING_AUDIO_SECOND_MS = 1500;
|
||||
const FINAL_TIMEOUT_PER_MISSING_SEQ_MS = 250;
|
||||
const SILENCE_PEAK_THRESHOLD = 300;
|
||||
|
||||
const secondsToPcm16Bytes = (seconds, sampleRate) =>
|
||||
seconds > 0 ? Math.max(1, Math.round(seconds * sampleRate * 2)) : 0;
|
||||
|
||||
/**
|
||||
* Split the current segment once it is long enough to be worth decoding on its
|
||||
* own and the speaker has just gone quiet, or unconditionally at the hard cap.
|
||||
* Client chunks are ~1s, so a quiet chunk is roughly a second of silence — long
|
||||
* enough to be a sentence boundary rather than a gap between words.
|
||||
*/
|
||||
function shouldSplitSegment(state) {
|
||||
if (state.segmentMaxBytes > 0 && state.bytesSinceCommit >= state.segmentMaxBytes) {
|
||||
return true;
|
||||
}
|
||||
if (state.segmentMinBytes <= 0 || state.bytesSinceCommit < state.segmentMinBytes) {
|
||||
return false;
|
||||
}
|
||||
return state.lastChunkPeak < SILENCE_PEAK_THRESHOLD;
|
||||
}
|
||||
|
||||
export class DictationStreamManager {
|
||||
/**
|
||||
* @param {object} params
|
||||
@@ -33,13 +64,15 @@ export class DictationStreamManager {
|
||||
* The streaming transcription session contract:
|
||||
* { requiredSampleRate, appendPcm16(buf), commit(), clear(), close(), on(event, handler) }
|
||||
* @param {number} [params.finalTimeoutMs]
|
||||
* @param {number} [params.autoCommitSeconds]
|
||||
* @param {number} [params.segmentMinSeconds] audio before a pause may split a segment
|
||||
* @param {number} [params.segmentMaxSeconds] hard segment cap for pauseless speech
|
||||
*/
|
||||
constructor({ emit, createSttSession, finalTimeoutMs, autoCommitSeconds }) {
|
||||
constructor({ emit, createSttSession, finalTimeoutMs, segmentMinSeconds, segmentMaxSeconds }) {
|
||||
this.emit = emit;
|
||||
this.createSttSession = createSttSession;
|
||||
this.finalTimeoutMs = finalTimeoutMs ?? DEFAULT_FINAL_TIMEOUT_MS;
|
||||
this.autoCommitSeconds = autoCommitSeconds ?? DEFAULT_AUTO_COMMIT_SECONDS;
|
||||
this.segmentMinSeconds = segmentMinSeconds ?? DEFAULT_SEGMENT_MIN_SECONDS;
|
||||
this.segmentMaxSeconds = segmentMaxSeconds ?? DEFAULT_SEGMENT_MAX_SECONDS;
|
||||
this.streams = new Map();
|
||||
}
|
||||
|
||||
@@ -87,13 +120,12 @@ export class DictationStreamManager {
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
// Segment accounting is reset where the commit is issued, not here: this
|
||||
// event arrives after an async hop, and zeroing the counters on arrival
|
||||
// would discard audio that came in meanwhile — up to and including
|
||||
// mistaking the tail of the dictation for silence and clearing it.
|
||||
state.committedSegmentIds.push(segmentId);
|
||||
state.bytesSinceCommit = 0;
|
||||
state.peakSinceCommit = 0;
|
||||
|
||||
if (state.finishRequested && state.awaitingFinalCommit) {
|
||||
state.awaitingFinalCommit = false;
|
||||
}
|
||||
state.pendingCommits = Math.max(0, state.pendingCommits - 1);
|
||||
|
||||
this.maybeFinalizeStream(dictationId);
|
||||
});
|
||||
@@ -108,10 +140,6 @@ export class DictationStreamManager {
|
||||
state.finalTranscriptSegmentIds.add(segmentId);
|
||||
}
|
||||
|
||||
if (state.finishRequested && state.awaitingFinalCommit && isFinal) {
|
||||
state.awaitingFinalCommit = false;
|
||||
}
|
||||
|
||||
const orderedIds = state.committedSegmentIds.includes(segmentId)
|
||||
? state.committedSegmentIds
|
||||
: [...state.committedSegmentIds, segmentId];
|
||||
@@ -143,16 +171,15 @@ export class DictationStreamManager {
|
||||
receivedChunks: new Map(),
|
||||
nextSeqToForward: 0,
|
||||
ackSeq: -1,
|
||||
autoCommitBytes:
|
||||
this.autoCommitSeconds > 0
|
||||
? Math.max(1, Math.round(this.autoCommitSeconds * stt.requiredSampleRate * 2))
|
||||
: 0,
|
||||
segmentMinBytes: secondsToPcm16Bytes(this.segmentMinSeconds, stt.requiredSampleRate),
|
||||
segmentMaxBytes: secondsToPcm16Bytes(this.segmentMaxSeconds, stt.requiredSampleRate),
|
||||
bytesSinceCommit: 0,
|
||||
peakSinceCommit: 0,
|
||||
lastChunkPeak: 0,
|
||||
committedSegmentIds: [],
|
||||
transcriptsBySegmentId: new Map(),
|
||||
finalTranscriptSegmentIds: new Set(),
|
||||
awaitingFinalCommit: false,
|
||||
pendingCommits: 0,
|
||||
finishRequested: false,
|
||||
finishSealed: false,
|
||||
finalSeq: null,
|
||||
@@ -203,7 +230,8 @@ export class DictationStreamManager {
|
||||
if (resampled.length > 0) {
|
||||
state.stt.appendPcm16(resampled);
|
||||
state.bytesSinceCommit += resampled.length;
|
||||
state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled));
|
||||
state.lastChunkPeak = pcm16lePeakAbs(resampled);
|
||||
state.peakSinceCommit = Math.max(state.peakSinceCommit, state.lastChunkPeak);
|
||||
try {
|
||||
this.maybeAutoCommitSegment(state);
|
||||
} catch (error) {
|
||||
@@ -325,9 +353,7 @@ export class DictationStreamManager {
|
||||
return state.finalTranscriptSegmentIds.has(segmentId) ? count : count + 1;
|
||||
}, 0);
|
||||
const pendingSegments =
|
||||
pendingCommittedSegments +
|
||||
pendingUncommittedTranscriptSegments +
|
||||
(state.awaitingFinalCommit ? 1 : 0);
|
||||
pendingCommittedSegments + pendingUncommittedTranscriptSegments + state.pendingCommits;
|
||||
const pendingAudioSeconds = Math.ceil(Math.max(0, state.bytesSinceCommit) / bytesPerSecond);
|
||||
const missingSeqCount =
|
||||
state.finalSeq === null ? 0 : Math.max(0, state.finalSeq - state.ackSeq);
|
||||
@@ -347,19 +373,36 @@ export class DictationStreamManager {
|
||||
if (state.finishRequested) {
|
||||
return;
|
||||
}
|
||||
if (state.autoCommitBytes <= 0 || state.bytesSinceCommit < state.autoCommitBytes) {
|
||||
if (!shouldSplitSegment(state)) {
|
||||
return;
|
||||
}
|
||||
if (state.peakSinceCommit < SILENCE_PEAK_THRESHOLD) {
|
||||
state.stt.clear();
|
||||
state.bytesSinceCommit = 0;
|
||||
state.peakSinceCommit = 0;
|
||||
state.lastChunkPeak = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
state.bytesSinceCommit = 0;
|
||||
state.peakSinceCommit = 0;
|
||||
state.stt.commit();
|
||||
state.lastChunkPeak = 0;
|
||||
this.commitSegment(state);
|
||||
}
|
||||
|
||||
/**
|
||||
* Issue a commit and record it as in flight. The session acknowledges with a
|
||||
* `committed` event; until then the manager must not finalize, or the
|
||||
* segment's transcript would be missing from the final text.
|
||||
*/
|
||||
commitSegment(state) {
|
||||
state.pendingCommits += 1;
|
||||
try {
|
||||
state.stt.commit();
|
||||
} catch (error) {
|
||||
state.pendingCommits -= 1;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
maybeSealStreamFinish(dictationId) {
|
||||
@@ -382,19 +425,19 @@ export class DictationStreamManager {
|
||||
state.stt.clear();
|
||||
state.bytesSinceCommit = 0;
|
||||
state.peakSinceCommit = 0;
|
||||
state.awaitingFinalCommit = false;
|
||||
state.lastChunkPeak = 0;
|
||||
this.dropUncommittedNonFinalTranscripts(state);
|
||||
} else {
|
||||
state.awaitingFinalCommit = true;
|
||||
state.bytesSinceCommit = 0;
|
||||
state.peakSinceCommit = 0;
|
||||
state.lastChunkPeak = 0;
|
||||
try {
|
||||
state.stt.commit();
|
||||
this.commitSegment(state);
|
||||
} catch (error) {
|
||||
this.failAndCleanupStream(dictationId, error?.message || String(error), true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
state.awaitingFinalCommit = false;
|
||||
}
|
||||
|
||||
state.finishSealed = true;
|
||||
@@ -425,7 +468,7 @@ export class DictationStreamManager {
|
||||
if (state.ackSeq < state.finalSeq) {
|
||||
return;
|
||||
}
|
||||
if (state.awaitingFinalCommit) {
|
||||
if (state.pendingCommits > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -175,8 +175,8 @@ describe('DictationStreamManager', () => {
|
||||
},
|
||||
});
|
||||
const { manager, messages } = createManager(session);
|
||||
// Force auto-commit after ~0.05s of audio so two segments form.
|
||||
manager.autoCommitSeconds = 0.05;
|
||||
// Force a hard-cap split after ~0.05s of audio so two segments form.
|
||||
manager.segmentMaxSeconds = 0.05;
|
||||
|
||||
await manager.handleStart('d1', FORMAT, {});
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(1600) });
|
||||
@@ -191,4 +191,62 @@ describe('DictationStreamManager', () => {
|
||||
const partials = messages.filter((m) => m.type === 'partial');
|
||||
expect(partials.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('keeps a short dictation as one segment even across pauses', async () => {
|
||||
const session = new FakeSttSession();
|
||||
const { manager } = createManager(session);
|
||||
|
||||
await manager.handleStart('d1', FORMAT, {});
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(16000) });
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: silentChunkBase64(16000) });
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 2, audioBase64: loudChunkBase64(16000) });
|
||||
|
||||
expect(session.commits).toBe(0);
|
||||
|
||||
manager.handleFinish('d1', 2);
|
||||
await waitFor(() => session.commits === 1);
|
||||
});
|
||||
|
||||
it('splits at a pause once the segment passes the minimum length', async () => {
|
||||
const session = new FakeSttSession();
|
||||
const { manager } = createManager(session);
|
||||
manager.segmentMinSeconds = 3;
|
||||
|
||||
await manager.handleStart('d1', FORMAT, {});
|
||||
// 2s of audio: below the minimum, so this pause must not split.
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(16000) });
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: silentChunkBase64(16000) });
|
||||
expect(session.commits).toBe(0);
|
||||
|
||||
// Past the minimum, the next quiet chunk is a segment boundary.
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 2, audioBase64: loudChunkBase64(16000) });
|
||||
expect(session.commits).toBe(0);
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 3, audioBase64: silentChunkBase64(16000) });
|
||||
expect(session.commits).toBe(1);
|
||||
});
|
||||
|
||||
it('splits pauseless speech at the hard cap', async () => {
|
||||
const session = new FakeSttSession();
|
||||
const { manager } = createManager(session);
|
||||
manager.segmentMinSeconds = 60;
|
||||
manager.segmentMaxSeconds = 2;
|
||||
|
||||
await manager.handleStart('d1', FORMAT, {});
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(16000) });
|
||||
expect(session.commits).toBe(0);
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: loudChunkBase64(16000) });
|
||||
expect(session.commits).toBe(1);
|
||||
});
|
||||
|
||||
it('clears a silence-only segment at the hard cap instead of committing it', async () => {
|
||||
const session = new FakeSttSession();
|
||||
const { manager } = createManager(session);
|
||||
manager.segmentMaxSeconds = 1;
|
||||
|
||||
await manager.handleStart('d1', FORMAT, {});
|
||||
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: silentChunkBase64(16000) });
|
||||
|
||||
expect(session.commits).toBe(0);
|
||||
expect(session.clears).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user