+
{status === 'recording' ? (
<>
() => void;
+ className?: string;
+}
+
+const BAR_WIDTH = 2;
+const BAR_GAP = 3;
+const BAR_PITCH = BAR_WIDTH + BAR_GAP;
+/** One sample per bar; ~16 bars/s scrolls at a readable speed. */
+const SAMPLE_INTERVAL_MS = 60;
+/** Raise quiet speech so normal talking uses most of the height. */
+const LEVEL_CURVE = 0.65;
+
+export const DictationWaveform: React.FC = ({ subscribeLevel, className }) => {
+ const { currentTheme } = useThemeSystem();
+ const canvasRef = React.useRef(null);
+ const barColor = currentTheme.colors.surface.mutedForeground;
+
+ React.useEffect(() => {
+ const canvas = canvasRef.current;
+ if (!canvas) {
+ return;
+ }
+ const context = canvas.getContext('2d');
+ if (!context) {
+ return;
+ }
+
+ // Peak-hold between samples: a short loud syllable must not be missed
+ // just because it landed between two frames.
+ let peakSinceSample = 0;
+ const unsubscribe = subscribeLevel((level) => {
+ peakSinceSample = Math.max(peakSinceSample, level);
+ });
+
+ const bars: number[] = [];
+ let cssWidth = 0;
+ let cssHeight = 0;
+
+ const resize = () => {
+ const rect = canvas.getBoundingClientRect();
+ const dpr = window.devicePixelRatio || 1;
+ cssWidth = rect.width;
+ cssHeight = rect.height;
+ canvas.width = Math.max(1, Math.round(cssWidth * dpr));
+ canvas.height = Math.max(1, Math.round(cssHeight * dpr));
+ context.setTransform(dpr, 0, 0, dpr, 0, 0);
+ };
+ resize();
+ const observer = new ResizeObserver(resize);
+ observer.observe(canvas);
+
+ const draw = () => {
+ const capacity = Math.max(1, Math.floor(cssWidth / BAR_PITCH));
+ while (bars.length > capacity) {
+ bars.shift();
+ }
+
+ context.clearRect(0, 0, cssWidth, cssHeight);
+ context.strokeStyle = barColor;
+ context.fillStyle = barColor;
+ context.lineWidth = BAR_WIDTH;
+ context.lineCap = 'round';
+
+ const centerY = cssHeight / 2;
+ const maxHeight = Math.max(BAR_WIDTH, cssHeight);
+ // Anchor the newest bar to the right edge; older bars trail left.
+ const rightX = cssWidth - BAR_WIDTH / 2;
+
+ for (let i = 0; i < bars.length; i++) {
+ const x = rightX - (bars.length - 1 - i) * BAR_PITCH;
+ if (x < BAR_WIDTH / 2) {
+ continue;
+ }
+ const height = BAR_WIDTH + (maxHeight - BAR_WIDTH) * Math.pow(bars[i], LEVEL_CURVE);
+ // Round caps add BAR_WIDTH/2 past each end of the stroke, so the
+ // stroke itself is the height minus one cap diameter.
+ const half = (height - BAR_WIDTH) / 2;
+ context.beginPath();
+ if (half < 0.25) {
+ context.arc(x, centerY, BAR_WIDTH / 2, 0, Math.PI * 2);
+ context.fill();
+ } else {
+ context.moveTo(x, centerY - half);
+ context.lineTo(x, centerY + half);
+ context.stroke();
+ }
+ }
+ };
+
+ let frame = 0;
+ let lastSampleAt = 0;
+ const tick = (now: number) => {
+ frame = requestAnimationFrame(tick);
+ if (now - lastSampleAt < SAMPLE_INTERVAL_MS) {
+ return;
+ }
+ lastSampleAt = now;
+ bars.push(peakSinceSample);
+ peakSinceSample = 0;
+ draw();
+ };
+ frame = requestAnimationFrame(tick);
+
+ return () => {
+ cancelAnimationFrame(frame);
+ observer.disconnect();
+ unsubscribe();
+ };
+ }, [subscribeLevel, barColor]);
+
+ return ;
stop: () => Promise;
- volume: number;
+ /** Subscribe to the normalized (0..1) mic level. Returns an unsubscribe. */
+ subscribeLevel: (listener: DictationLevelListener) => () => void;
}
const OUTPUT_RATE = 16000;
@@ -125,7 +132,18 @@ export const isDictationCaptureSupported = (): boolean => {
};
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
- const [volume, setVolume] = useState(0);
+ const levelListenersRef = useRef(new Set());
+ const emitLevel = useCallback((level: number) => {
+ for (const listener of levelListenersRef.current) {
+ listener(level);
+ }
+ }, []);
+ const subscribeLevel = useCallback((listener: DictationLevelListener) => {
+ levelListenersRef.current.add(listener);
+ return () => {
+ levelListenersRef.current.delete(listener);
+ };
+ }, []);
const onPcmSegmentRef = useRef(config.onPcmSegment);
const onErrorRef = useRef(config.onError);
@@ -196,7 +214,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
sumSquares += input[i] * input[i];
}
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
- setVolume(Math.min(1, Math.max(0, rms * 2)));
+ emitLevel(Math.min(1, Math.max(0, rms * 2)));
const next = resampleToPcm16(input, context.sampleRate, OUTPUT_RATE);
graph.pending = concatInt16(graph.pending, next);
@@ -227,12 +245,12 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
graphRef.current = emptyGraph();
throw error instanceof Error ? error : new Error(String(error));
}
- }, []);
+ }, [emitLevel]);
const stop = useCallback(async () => {
const graph = graphRef.current;
graph.started = false;
- setVolume(0);
+ emitLevel(0);
if (graph.processor) {
try {
@@ -272,7 +290,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
if (graphRef.current === graph) {
graphRef.current = emptyGraph();
}
- }, []);
+ }, [emitLevel]);
useEffect(() => {
return () => {
@@ -294,8 +312,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
}
},
stop,
- volume,
+ subscribeLevel,
}),
- [start, stop, volume],
+ [start, stop, subscribeLevel],
);
}
diff --git a/packages/web/server/lib/dictation/DOCUMENTATION.md b/packages/web/server/lib/dictation/DOCUMENTATION.md
index d6a7940c..e66af116 100644
--- a/packages/web/server/lib/dictation/DOCUMENTATION.md
+++ b/packages/web/server/lib/dictation/DOCUMENTATION.md
@@ -1,9 +1,15 @@
# Dictation module
-Server-authoritative streaming speech-to-text for the chat composer, plus
-local text-to-speech. The client streams 16 kHz mono PCM16 chunks (base64)
-over a WebSocket; the server runs the transcription and streams live partial
-transcripts back.
+Server-authoritative speech-to-text for the chat composer, plus local
+text-to-speech. The client streams 16 kHz mono PCM16 chunks (base64) over a
+WebSocket while the user speaks; the server buffers them and transcribes each
+segment exactly once, when the segment is committed.
+
+Transcription is deliberately not incremental. Parakeet is an offline model
+trained on whole utterances, so re-decoding the growing buffer to animate a
+live transcript costs O(n^2) work for a result the final decode replaces. The
+composer shows no text while recording and inserts the full transcript on
+stop.
Local TTS (Kokoro via sherpa-onnx OfflineTts) runs in the same worker process
and is exposed as `POST /api/dictation/tts/speak` (JSON `{text, speakerId?,
@@ -21,9 +27,9 @@ same status/download/delete routes.
Created from the startup pipeline (`startup-pipeline-runtime.js`) before
the generic OpenCode proxy so routes are not shadowed.
- `stream-manager.js` — `DictationStreamManager`, one per WS connection.
- Chunk reordering by `seq` + ack, resampling to the provider rate,
- auto-commit every ~15 s of audio, silence suppression by PCM peak,
- partial-transcript concatenation, adaptive finalization timeout.
+ Chunk reordering by `seq` + ack, resampling to the provider rate, segment
+ splitting, silence suppression by PCM peak, partial-transcript
+ concatenation, adaptive finalization timeout.
- `service.js` — provider resolution and readiness. Providers:
- `local` (default): sherpa-onnx Parakeet TDT in a forked worker process.
Models auto-download in the background on first use; while missing, the
@@ -33,7 +39,7 @@ same status/download/delete routes.
OpenAI-compatible `/v1/audio/transcriptions` endpoint
(`openai-compatible-session.js`, reuses `../tts/stt.js`).
- `local/` — worker process + client (IPC, idle shutdown TTL), sherpa
- recognizer engine and realtime session (throttled re-decode for partials),
+ recognizer engine and segment session (one decode per committed segment),
model catalog and downloader. The native `sherpa-onnx-node` addon is only
ever loaded inside the worker process.
- `audio.js` — PCM16 helpers: format parsing, peak, WAV wrapping, streaming
@@ -53,9 +59,27 @@ Server → client: `ready`, `ack {ackSeq}`, `partial {text}`,
`{ provider: 'local' | 'openai-compatible', language?, localModel?,
openaiCompatible?: { baseUrl, model, apiKey } }`.
+## Segmentation
+
+A dictation is one segment unless it runs long. Past `segmentMinSeconds`
+(60 s) the manager commits on the first silent chunk, so cuts land at a pause
+rather than mid-word; `segmentMaxSeconds` (90 s) is a hard cap for speech with
+no pause in it. Client chunks are ~1 s, so "silent chunk" is roughly a second
+of silence.
+
+The bounds exist because Parakeet is a full-attention conformer: decode cost
+and peak memory grow quadratically with segment length. Measured on Parakeet
+v3 int8 with 2 threads: 60 s took 2.1 s and +90 MB, 180 s took 9.3 s and
++490 MB, 300 s took 21.3 s and +1.5 GB. Committed segments decode while the
+user is still speaking, so only the tail is left to transcribe on stop.
+
## Invariants
- Never load `sherpa-onnx-node` in the main server process.
+- Transcription happens on commit only; sessions never emit non-final
+ transcripts. The `partial` messages a client receives are the concatenation
+ of already-committed segments, and exist so a dictation that fails partway
+ can be salvaged instead of losing minutes of speech.
- The stream manager acks only the highest contiguous seq; the client is
expected to retain unacked segments for retry/replay.
- Silence-only segments (peak < 300) are cleared, never committed, so
diff --git a/packages/web/server/lib/dictation/local/sherpa-recognizer.js b/packages/web/server/lib/dictation/local/sherpa-recognizer.js
index 4a593f26..9eca5f66 100644
--- a/packages/web/server/lib/dictation/local/sherpa-recognizer.js
+++ b/packages/web/server/lib/dictation/local/sherpa-recognizer.js
@@ -1,7 +1,12 @@
/**
* Sherpa-onnx offline recognizer engine (NeMo transducer / Parakeet) plus a
- * realtime streaming transcription session that re-decodes the accumulated
- * segment audio on a throttle to produce live partial transcripts.
+ * segment transcription session that decodes each segment exactly once, when
+ * the segment is committed.
+ *
+ * Parakeet is an offline model: it is trained to see a whole utterance at
+ * once. Decoding the accumulated audio repeatedly to animate a live transcript
+ * costs O(n^2) work for a result the final decode throws away, so this session
+ * only decodes on commit.
*
* Runs inside the dictation worker process only — never load the native
* addon in the main server process.
@@ -147,31 +152,26 @@ export class SherpaOfflineRecognizerEngine {
}
/**
- * Streaming transcription session backed by the offline recognizer.
- * Accumulates the current segment's PCM and re-decodes it at most every
- * `minDecodeIntervalMs` to emit non-final partial transcripts; `commit()`
- * finalizes the segment and starts a new one.
+ * Segment transcription session backed by the offline recognizer.
+ * Accumulates the current segment's PCM and decodes it once in `commit()`,
+ * which emits the segment's final transcript and starts a new segment.
*
* Implements the StreamingTranscriptionSession contract used by
- * DictationStreamManager.
+ * DictationStreamManager. It never emits non-final transcripts: the manager's
+ * live `partial` messages are the concatenation of already-committed segments.
*/
-export class SherpaRealtimeTranscriptionSession extends EventEmitter {
+export class SherpaSegmentTranscriptionSession extends EventEmitter {
/**
- * @param {{ engine: SherpaOfflineRecognizerEngine, minDecodeIntervalMs?: number }} params
+ * @param {{ engine: SherpaOfflineRecognizerEngine }} params
*/
- constructor({ engine, minDecodeIntervalMs }) {
+ constructor({ engine }) {
super();
this.engine = engine;
this.requiredSampleRate = engine.sampleRate;
- this.minDecodeIntervalMs = minDecodeIntervalMs ?? 350;
this.connected = false;
this.currentSegmentId = null;
this.previousSegmentId = null;
- this.lastPartialText = '';
this.pcm16 = Buffer.alloc(0);
- this.lastDecodeAt = 0;
- this.decoding = false;
- this.pendingDecode = false;
}
async connect() {
@@ -184,39 +184,38 @@ export class SherpaRealtimeTranscriptionSession extends EventEmitter {
appendPcm16(chunk) {
if (!this.connected || !this.currentSegmentId) {
- this.emit('error', new Error('Sherpa realtime session not connected'));
+ this.emit('error', new Error('Sherpa transcription session not connected'));
return;
}
this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
- this.maybeDecode(false).catch((err) => {
- this.emit('error', err instanceof Error ? err : new Error(String(err)));
- });
}
commit() {
if (!this.connected || !this.currentSegmentId) {
- this.emit('error', new Error('Sherpa realtime session not connected'));
+ this.emit('error', new Error('Sherpa transcription session not connected'));
return;
}
- void (async () => {
- try {
- await this.maybeDecode(true);
- const finalText = this.lastPartialText;
- const segmentId = this.currentSegmentId;
- const previousSegmentId = this.previousSegmentId;
+ const segmentId = this.currentSegmentId;
+ const previousSegmentId = this.previousSegmentId;
+ const pcm16 = this.pcm16;
- this.emit('committed', { segmentId, previousSegmentId });
- this.emit('transcript', { segmentId, transcript: finalText, isFinal: true });
+ // Start the next segment before decoding: decoding blocks the worker for
+ // seconds on long segments, and audio for the next one keeps arriving.
+ this.previousSegmentId = segmentId;
+ this.currentSegmentId = randomUUID();
+ this.pcm16 = Buffer.alloc(0);
- this.previousSegmentId = segmentId;
- this.currentSegmentId = randomUUID();
- this.lastPartialText = '';
- this.pcm16 = Buffer.alloc(0);
- } catch (err) {
- this.emit('error', err instanceof Error ? err : new Error(String(err)));
- }
- })();
+ this.emit('committed', { segmentId, previousSegmentId });
+
+ let transcript;
+ try {
+ transcript = this.engine.decodePcm16(pcm16);
+ } catch (err) {
+ this.emit('error', err instanceof Error ? err : new Error(String(err)));
+ return;
+ }
+ this.emit('transcript', { segmentId, transcript, isFinal: true });
}
clear() {
@@ -225,7 +224,6 @@ export class SherpaRealtimeTranscriptionSession extends EventEmitter {
}
this.pcm16 = Buffer.alloc(0);
this.currentSegmentId = randomUUID();
- this.lastPartialText = '';
}
close() {
@@ -233,45 +231,4 @@ export class SherpaRealtimeTranscriptionSession extends EventEmitter {
this.currentSegmentId = null;
this.pcm16 = Buffer.alloc(0);
}
-
- async maybeDecode(force) {
- if (!this.connected || !this.currentSegmentId) {
- return;
- }
-
- const now = Date.now();
- if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) {
- return;
- }
-
- if (this.decoding) {
- this.pendingDecode = true;
- return;
- }
-
- this.decoding = true;
- try {
- const decodeStartedAt = Date.now();
- const text = this.engine.decodePcm16(this.pcm16);
- this.lastDecodeAt = Date.now();
- // Adaptive throttle: on slow hardware (or heavy models) re-decoding the
- // growing segment every 350ms would monopolize the worker. Space partial
- // decodes to ~1.5x the observed decode time.
- this.minDecodeIntervalMs = Math.max(350, (this.lastDecodeAt - decodeStartedAt) * 1.5);
- if (text !== this.lastPartialText) {
- this.lastPartialText = text;
- this.emit('transcript', {
- segmentId: this.currentSegmentId,
- transcript: text,
- isFinal: false,
- });
- }
- } finally {
- this.decoding = false;
- if (this.pendingDecode) {
- this.pendingDecode = false;
- await this.maybeDecode(true);
- }
- }
- }
}
diff --git a/packages/web/server/lib/dictation/local/worker-process.js b/packages/web/server/lib/dictation/local/worker-process.js
index 2629cc9e..3a5c34b8 100644
--- a/packages/web/server/lib/dictation/local/worker-process.js
+++ b/packages/web/server/lib/dictation/local/worker-process.js
@@ -17,7 +17,7 @@
import {
SherpaOfflineRecognizerEngine,
- SherpaRealtimeTranscriptionSession,
+ SherpaSegmentTranscriptionSession,
} from './sherpa-recognizer.js';
import { SherpaTtsEngine } from './sherpa-tts.js';
import { getLocalSttModelDir, getLocalSttModelSpec } from './model-catalog.js';
@@ -126,7 +126,7 @@ async function handleRequest(message) {
case 'session.create': {
cleanupSession(message.sessionId);
const engine = getEngine(message.modelsDir, message.modelId);
- const session = new SherpaRealtimeTranscriptionSession({ engine });
+ const session = new SherpaSegmentTranscriptionSession({ engine });
session.on('committed', (payload) => {
sendToParent({ type: 'session.committed', sessionId: message.sessionId, payload });
});
diff --git a/packages/web/server/lib/dictation/openai-compatible-session.js b/packages/web/server/lib/dictation/openai-compatible-session.js
index d4e1bc7f..3be98ef6 100644
--- a/packages/web/server/lib/dictation/openai-compatible-session.js
+++ b/packages/web/server/lib/dictation/openai-compatible-session.js
@@ -3,8 +3,9 @@
* endpoints (faster-whisper, whisper.cpp, OpenAI, ...).
*
* The Whisper HTTP API cannot stream, so audio is buffered per segment and
- * transcribed on commit(). Live partials therefore only advance at segment
- * boundaries (the DictationStreamManager auto-commits every ~15s of speech).
+ * transcribed on commit(). This matches how the local session behaves: the
+ * DictationStreamManager splits long dictations at pauses, and everything
+ * shorter is one request on stop.
*
* Implements the StreamingTranscriptionSession contract used by
* DictationStreamManager.
diff --git a/packages/web/server/lib/dictation/stream-manager.js b/packages/web/server/lib/dictation/stream-manager.js
index 776d7a8d..b4c595cb 100644
--- a/packages/web/server/lib/dictation/stream-manager.js
+++ b/packages/web/server/lib/dictation/stream-manager.js
@@ -7,23 +7,54 @@
* Responsibilities:
* - Reorders inbound chunks by `seq` and acks the highest contiguous seq.
* - Resamples client PCM (16 kHz by default) to the provider's required rate.
- * - Auto-commits a segment every `autoCommitSeconds` of audio, but clears
- * silence-only segments instead of committing them.
+ * - Segments long dictations at natural pauses: past `segmentMinSeconds` of
+ * audio it commits on the first silent chunk, and `segmentMaxSeconds` is a
+ * hard cap for speech with no pause in it. Silence-only segments are
+ * cleared instead of committed.
* - Concatenates per-segment transcripts into live partials and emits the
- * final text once every committed segment has a final transcript.
+ * final text once every committed segment has a final transcript. The
+ * manager counts the commits it issued rather than trusting the session's
+ * echoed events, so a commit still in flight when the client finishes
+ * cannot be silently dropped from the transcript.
* - Applies an adaptive finalization timeout budget based on pending work.
*/
import { Pcm16MonoResampler, parsePcmRateFromFormat, pcm16lePeakAbs } from './audio.js';
const DEFAULT_FINAL_TIMEOUT_MS = 10000;
-const DEFAULT_AUTO_COMMIT_SECONDS = 15;
+// Parakeet is a full-attention conformer: decode cost and peak memory grow
+// quadratically with segment length (measured: 60s -> 2.1s/+90MB,
+// 300s -> 21.3s/+1.5GB). Segmenting keeps a long dictation off that curve and
+// lets committed segments decode while the user is still speaking, so only the
+// tail is left to transcribe on stop. Typical dictations are shorter than the
+// minimum and are decoded as a single segment.
+const DEFAULT_SEGMENT_MIN_SECONDS = 60;
+const DEFAULT_SEGMENT_MAX_SECONDS = 90;
const FINAL_TIMEOUT_MAX_MS = 5 * 60 * 1000;
const FINAL_TIMEOUT_PER_PENDING_SEGMENT_MS = 15 * 1000;
const FINAL_TIMEOUT_PER_PENDING_AUDIO_SECOND_MS = 1500;
const FINAL_TIMEOUT_PER_MISSING_SEQ_MS = 250;
const SILENCE_PEAK_THRESHOLD = 300;
+const secondsToPcm16Bytes = (seconds, sampleRate) =>
+ seconds > 0 ? Math.max(1, Math.round(seconds * sampleRate * 2)) : 0;
+
+/**
+ * Split the current segment once it is long enough to be worth decoding on its
+ * own and the speaker has just gone quiet, or unconditionally at the hard cap.
+ * Client chunks are ~1s, so a quiet chunk is roughly a second of silence — long
+ * enough to be a sentence boundary rather than a gap between words.
+ */
+function shouldSplitSegment(state) {
+ if (state.segmentMaxBytes > 0 && state.bytesSinceCommit >= state.segmentMaxBytes) {
+ return true;
+ }
+ if (state.segmentMinBytes <= 0 || state.bytesSinceCommit < state.segmentMinBytes) {
+ return false;
+ }
+ return state.lastChunkPeak < SILENCE_PEAK_THRESHOLD;
+}
+
export class DictationStreamManager {
/**
* @param {object} params
@@ -33,13 +64,15 @@ export class DictationStreamManager {
* The streaming transcription session contract:
* { requiredSampleRate, appendPcm16(buf), commit(), clear(), close(), on(event, handler) }
* @param {number} [params.finalTimeoutMs]
- * @param {number} [params.autoCommitSeconds]
+ * @param {number} [params.segmentMinSeconds] audio before a pause may split a segment
+ * @param {number} [params.segmentMaxSeconds] hard segment cap for pauseless speech
*/
- constructor({ emit, createSttSession, finalTimeoutMs, autoCommitSeconds }) {
+ constructor({ emit, createSttSession, finalTimeoutMs, segmentMinSeconds, segmentMaxSeconds }) {
this.emit = emit;
this.createSttSession = createSttSession;
this.finalTimeoutMs = finalTimeoutMs ?? DEFAULT_FINAL_TIMEOUT_MS;
- this.autoCommitSeconds = autoCommitSeconds ?? DEFAULT_AUTO_COMMIT_SECONDS;
+ this.segmentMinSeconds = segmentMinSeconds ?? DEFAULT_SEGMENT_MIN_SECONDS;
+ this.segmentMaxSeconds = segmentMaxSeconds ?? DEFAULT_SEGMENT_MAX_SECONDS;
this.streams = new Map();
}
@@ -87,13 +120,12 @@ export class DictationStreamManager {
if (!state) {
return;
}
+ // Segment accounting is reset where the commit is issued, not here: this
+ // event arrives after an async hop, and zeroing the counters on arrival
+ // would discard audio that came in meanwhile — up to and including
+ // mistaking the tail of the dictation for silence and clearing it.
state.committedSegmentIds.push(segmentId);
- state.bytesSinceCommit = 0;
- state.peakSinceCommit = 0;
-
- if (state.finishRequested && state.awaitingFinalCommit) {
- state.awaitingFinalCommit = false;
- }
+ state.pendingCommits = Math.max(0, state.pendingCommits - 1);
this.maybeFinalizeStream(dictationId);
});
@@ -108,10 +140,6 @@ export class DictationStreamManager {
state.finalTranscriptSegmentIds.add(segmentId);
}
- if (state.finishRequested && state.awaitingFinalCommit && isFinal) {
- state.awaitingFinalCommit = false;
- }
-
const orderedIds = state.committedSegmentIds.includes(segmentId)
? state.committedSegmentIds
: [...state.committedSegmentIds, segmentId];
@@ -143,16 +171,15 @@ export class DictationStreamManager {
receivedChunks: new Map(),
nextSeqToForward: 0,
ackSeq: -1,
- autoCommitBytes:
- this.autoCommitSeconds > 0
- ? Math.max(1, Math.round(this.autoCommitSeconds * stt.requiredSampleRate * 2))
- : 0,
+ segmentMinBytes: secondsToPcm16Bytes(this.segmentMinSeconds, stt.requiredSampleRate),
+ segmentMaxBytes: secondsToPcm16Bytes(this.segmentMaxSeconds, stt.requiredSampleRate),
bytesSinceCommit: 0,
peakSinceCommit: 0,
+ lastChunkPeak: 0,
committedSegmentIds: [],
transcriptsBySegmentId: new Map(),
finalTranscriptSegmentIds: new Set(),
- awaitingFinalCommit: false,
+ pendingCommits: 0,
finishRequested: false,
finishSealed: false,
finalSeq: null,
@@ -203,7 +230,8 @@ export class DictationStreamManager {
if (resampled.length > 0) {
state.stt.appendPcm16(resampled);
state.bytesSinceCommit += resampled.length;
- state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled));
+ state.lastChunkPeak = pcm16lePeakAbs(resampled);
+ state.peakSinceCommit = Math.max(state.peakSinceCommit, state.lastChunkPeak);
try {
this.maybeAutoCommitSegment(state);
} catch (error) {
@@ -325,9 +353,7 @@ export class DictationStreamManager {
return state.finalTranscriptSegmentIds.has(segmentId) ? count : count + 1;
}, 0);
const pendingSegments =
- pendingCommittedSegments +
- pendingUncommittedTranscriptSegments +
- (state.awaitingFinalCommit ? 1 : 0);
+ pendingCommittedSegments + pendingUncommittedTranscriptSegments + state.pendingCommits;
const pendingAudioSeconds = Math.ceil(Math.max(0, state.bytesSinceCommit) / bytesPerSecond);
const missingSeqCount =
state.finalSeq === null ? 0 : Math.max(0, state.finalSeq - state.ackSeq);
@@ -347,19 +373,36 @@ export class DictationStreamManager {
if (state.finishRequested) {
return;
}
- if (state.autoCommitBytes <= 0 || state.bytesSinceCommit < state.autoCommitBytes) {
+ if (!shouldSplitSegment(state)) {
return;
}
if (state.peakSinceCommit < SILENCE_PEAK_THRESHOLD) {
state.stt.clear();
state.bytesSinceCommit = 0;
state.peakSinceCommit = 0;
+ state.lastChunkPeak = 0;
return;
}
state.bytesSinceCommit = 0;
state.peakSinceCommit = 0;
- state.stt.commit();
+ state.lastChunkPeak = 0;
+ this.commitSegment(state);
+ }
+
+ /**
+ * Issue a commit and record it as in flight. The session acknowledges with a
+ * `committed` event; until then the manager must not finalize, or the
+ * segment's transcript would be missing from the final text.
+ */
+ commitSegment(state) {
+ state.pendingCommits += 1;
+ try {
+ state.stt.commit();
+ } catch (error) {
+ state.pendingCommits -= 1;
+ throw error;
+ }
}
maybeSealStreamFinish(dictationId) {
@@ -382,19 +425,19 @@ export class DictationStreamManager {
state.stt.clear();
state.bytesSinceCommit = 0;
state.peakSinceCommit = 0;
- state.awaitingFinalCommit = false;
+ state.lastChunkPeak = 0;
this.dropUncommittedNonFinalTranscripts(state);
} else {
- state.awaitingFinalCommit = true;
+ state.bytesSinceCommit = 0;
+ state.peakSinceCommit = 0;
+ state.lastChunkPeak = 0;
try {
- state.stt.commit();
+ this.commitSegment(state);
} catch (error) {
this.failAndCleanupStream(dictationId, error?.message || String(error), true);
return;
}
}
- } else {
- state.awaitingFinalCommit = false;
}
state.finishSealed = true;
@@ -425,7 +468,7 @@ export class DictationStreamManager {
if (state.ackSeq < state.finalSeq) {
return;
}
- if (state.awaitingFinalCommit) {
+ if (state.pendingCommits > 0) {
return;
}
diff --git a/packages/web/server/lib/dictation/stream-manager.test.js b/packages/web/server/lib/dictation/stream-manager.test.js
index a9e5660b..110c9ddb 100644
--- a/packages/web/server/lib/dictation/stream-manager.test.js
+++ b/packages/web/server/lib/dictation/stream-manager.test.js
@@ -175,8 +175,8 @@ describe('DictationStreamManager', () => {
},
});
const { manager, messages } = createManager(session);
- // Force auto-commit after ~0.05s of audio so two segments form.
- manager.autoCommitSeconds = 0.05;
+ // Force a hard-cap split after ~0.05s of audio so two segments form.
+ manager.segmentMaxSeconds = 0.05;
await manager.handleStart('d1', FORMAT, {});
manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(1600) });
@@ -191,4 +191,62 @@ describe('DictationStreamManager', () => {
const partials = messages.filter((m) => m.type === 'partial');
expect(partials.length).toBeGreaterThan(0);
});
+
+ it('keeps a short dictation as one segment even across pauses', async () => {
+ const session = new FakeSttSession();
+ const { manager } = createManager(session);
+
+ await manager.handleStart('d1', FORMAT, {});
+ manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(16000) });
+ manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: silentChunkBase64(16000) });
+ manager.handleChunk({ dictationId: 'd1', seq: 2, audioBase64: loudChunkBase64(16000) });
+
+ expect(session.commits).toBe(0);
+
+ manager.handleFinish('d1', 2);
+ await waitFor(() => session.commits === 1);
+ });
+
+ it('splits at a pause once the segment passes the minimum length', async () => {
+ const session = new FakeSttSession();
+ const { manager } = createManager(session);
+ manager.segmentMinSeconds = 3;
+
+ await manager.handleStart('d1', FORMAT, {});
+ // 2s of audio: below the minimum, so this pause must not split.
+ manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(16000) });
+ manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: silentChunkBase64(16000) });
+ expect(session.commits).toBe(0);
+
+ // Past the minimum, the next quiet chunk is a segment boundary.
+ manager.handleChunk({ dictationId: 'd1', seq: 2, audioBase64: loudChunkBase64(16000) });
+ expect(session.commits).toBe(0);
+ manager.handleChunk({ dictationId: 'd1', seq: 3, audioBase64: silentChunkBase64(16000) });
+ expect(session.commits).toBe(1);
+ });
+
+ it('splits pauseless speech at the hard cap', async () => {
+ const session = new FakeSttSession();
+ const { manager } = createManager(session);
+ manager.segmentMinSeconds = 60;
+ manager.segmentMaxSeconds = 2;
+
+ await manager.handleStart('d1', FORMAT, {});
+ manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: loudChunkBase64(16000) });
+ expect(session.commits).toBe(0);
+ manager.handleChunk({ dictationId: 'd1', seq: 1, audioBase64: loudChunkBase64(16000) });
+ expect(session.commits).toBe(1);
+ });
+
+ it('clears a silence-only segment at the hard cap instead of committing it', async () => {
+ const session = new FakeSttSession();
+ const { manager } = createManager(session);
+ manager.segmentMaxSeconds = 1;
+
+ await manager.handleStart('d1', FORMAT, {});
+ manager.handleChunk({ dictationId: 'd1', seq: 0, audioBase64: silentChunkBase64(16000) });
+
+ expect(session.commits).toBe(0);
+ expect(session.clears).toBe(1);
+ });
});