feat(dictation): transcribe after recording instead of live
Parakeet is an offline model trained on whole utterances, so re-decoding the growing buffer to animate a live transcript cost O(n^2) work for a result the final decode replaced. Sessions now decode once per committed segment, and the composer shows a scrolling waveform of the mic level instead of running text. Long dictations split at a pause once past 60s (hard cap 90s) instead of on a blind 15s timer, so cuts no longer land mid-word. Committed segments decode while the user is still speaking: a 185s dictation returns 4.1s after stop instead of 11.0s, with identical text (816 vs 817 words). Also fixes two ways the stream manager could silently drop transcribed audio. It now counts the commits it issued instead of trusting the session's echoed events, so a commit still in flight when the client finishes can no longer be left out of the final text. And segment byte/peak accounting is reset where the commit is issued rather than when the event arrives, which could mistake the tail of a dictation for silence and clear it.
This commit is contained in:
@@ -5,6 +5,10 @@
|
||||
* area uses the same paddings/typography as the textarea and the action row
|
||||
* reuses the footer icon-button styling — so toggling dictation causes no
|
||||
* vertical shift.
|
||||
*
|
||||
* No text appears while recording. The server transcribes the audio once the
|
||||
* user stops, so the overlay shows the recording state and then Transcribing.
|
||||
* The only transcript rendered here is the salvage text of a failed dictation.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
@@ -15,6 +19,7 @@ import { useThemeSystem } from '@/contexts/useThemeSystem';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
import { useDictation } from '@/hooks/useDictation';
|
||||
import { DictationWaveform } from '@/components/dictation/DictationWaveform';
|
||||
import { isDictationCaptureSupported } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { isVSCodeRuntime } from '@/lib/desktop';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
@@ -50,25 +55,6 @@ const formatDuration = (seconds: number): string => {
|
||||
return `${mins}:${String(secs).padStart(2, '0')}`;
|
||||
};
|
||||
|
||||
const VolumeMeter: React.FC<{ volume: number }> = ({ volume }) => {
|
||||
const { currentTheme } = useThemeSystem();
|
||||
return (
|
||||
<div
|
||||
className="h-1.5 w-16 flex-shrink-0 overflow-hidden rounded-full"
|
||||
style={{ backgroundColor: currentTheme.colors.interactive.border }}
|
||||
aria-hidden="true"
|
||||
>
|
||||
<div
|
||||
className="h-full rounded-full transition-[width] duration-75"
|
||||
style={{
|
||||
width: `${Math.round(Math.min(1, volume) * 100)}%`,
|
||||
backgroundColor: currentTheme.colors.primary.base,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Polls the dictation status route while the local model is downloading and
|
||||
* returns the download percent (null while unknown / not downloading).
|
||||
@@ -162,7 +148,7 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
const {
|
||||
status,
|
||||
partialTranscript,
|
||||
volume,
|
||||
subscribeLevel,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
@@ -399,7 +385,7 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
{/* Measured for the composer-growth report — keep all
|
||||
transcript/placeholder/error content inside. */}
|
||||
<div ref={transcriptContentRef}>
|
||||
{partialTranscript ? (
|
||||
{status === 'failed' && partialTranscript ? (
|
||||
<p className="typography-markdown md:typography-ui-label whitespace-pre-wrap" style={{ color: currentTheme.colors.surface.foreground }}>
|
||||
{partialTranscript}
|
||||
</p>
|
||||
@@ -436,8 +422,8 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
style={{ backgroundColor: currentTheme.colors.status.error }}
|
||||
/>
|
||||
</span>
|
||||
<VolumeMeter volume={volume} />
|
||||
<span className="typography-meta tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||
<DictationWaveform subscribeLevel={subscribeLevel} className="block h-4 min-w-0 flex-1" />
|
||||
<span className="typography-meta flex-shrink-0 tabular-nums" style={{ color: currentTheme.colors.surface.mutedForeground }}>
|
||||
{formatDuration(duration)}
|
||||
</span>
|
||||
</>
|
||||
@@ -446,7 +432,7 @@ export const ComposerDictation: React.FC<ComposerDictationProps> = ({
|
||||
) : null}
|
||||
{/* Same inter-control gap as the composer's right cluster:
|
||||
gap-x-1 on mobile, md:gap-x-3 on desktop. */}
|
||||
<div className={cn('ml-auto flex items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
|
||||
<div className={cn('ml-auto flex flex-shrink-0 items-center', isMobile ? 'gap-x-1' : 'gap-x-1.5 md:gap-x-3')}>
|
||||
{status === 'recording' ? (
|
||||
<>
|
||||
<button
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Scrolling microphone level history for the dictation overlay.
|
||||
*
|
||||
* Newest sample is at the right edge and the history scrolls left, so the row
|
||||
* reads as a live recording trace rather than a single level bar. Silence
|
||||
* renders as a dot, speech as a rounded bar.
|
||||
*
|
||||
* Drawn on a canvas fed by a level subscription: the level updates ~12 times a
|
||||
* second, and routing that through React state would re-render the whole
|
||||
* dictation overlay at the same rate.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
|
||||
import { useThemeSystem } from '@/contexts/useThemeSystem';
|
||||
import type { DictationLevelListener } from '@/lib/dictation/use-dictation-audio-source';
|
||||
|
||||
interface DictationWaveformProps {
|
||||
subscribeLevel: (listener: DictationLevelListener) => () => void;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const BAR_WIDTH = 2;
|
||||
const BAR_GAP = 3;
|
||||
const BAR_PITCH = BAR_WIDTH + BAR_GAP;
|
||||
/** One sample per bar; ~16 bars/s scrolls at a readable speed. */
|
||||
const SAMPLE_INTERVAL_MS = 60;
|
||||
/** Raise quiet speech so normal talking uses most of the height. */
|
||||
const LEVEL_CURVE = 0.65;
|
||||
|
||||
export const DictationWaveform: React.FC<DictationWaveformProps> = ({ subscribeLevel, className }) => {
|
||||
const { currentTheme } = useThemeSystem();
|
||||
const canvasRef = React.useRef<HTMLCanvasElement | null>(null);
|
||||
const barColor = currentTheme.colors.surface.mutedForeground;
|
||||
|
||||
React.useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) {
|
||||
return;
|
||||
}
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Peak-hold between samples: a short loud syllable must not be missed
|
||||
// just because it landed between two frames.
|
||||
let peakSinceSample = 0;
|
||||
const unsubscribe = subscribeLevel((level) => {
|
||||
peakSinceSample = Math.max(peakSinceSample, level);
|
||||
});
|
||||
|
||||
const bars: number[] = [];
|
||||
let cssWidth = 0;
|
||||
let cssHeight = 0;
|
||||
|
||||
const resize = () => {
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
cssWidth = rect.width;
|
||||
cssHeight = rect.height;
|
||||
canvas.width = Math.max(1, Math.round(cssWidth * dpr));
|
||||
canvas.height = Math.max(1, Math.round(cssHeight * dpr));
|
||||
context.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
};
|
||||
resize();
|
||||
const observer = new ResizeObserver(resize);
|
||||
observer.observe(canvas);
|
||||
|
||||
const draw = () => {
|
||||
const capacity = Math.max(1, Math.floor(cssWidth / BAR_PITCH));
|
||||
while (bars.length > capacity) {
|
||||
bars.shift();
|
||||
}
|
||||
|
||||
context.clearRect(0, 0, cssWidth, cssHeight);
|
||||
context.strokeStyle = barColor;
|
||||
context.fillStyle = barColor;
|
||||
context.lineWidth = BAR_WIDTH;
|
||||
context.lineCap = 'round';
|
||||
|
||||
const centerY = cssHeight / 2;
|
||||
const maxHeight = Math.max(BAR_WIDTH, cssHeight);
|
||||
// Anchor the newest bar to the right edge; older bars trail left.
|
||||
const rightX = cssWidth - BAR_WIDTH / 2;
|
||||
|
||||
for (let i = 0; i < bars.length; i++) {
|
||||
const x = rightX - (bars.length - 1 - i) * BAR_PITCH;
|
||||
if (x < BAR_WIDTH / 2) {
|
||||
continue;
|
||||
}
|
||||
const height = BAR_WIDTH + (maxHeight - BAR_WIDTH) * Math.pow(bars[i], LEVEL_CURVE);
|
||||
// Round caps add BAR_WIDTH/2 past each end of the stroke, so the
|
||||
// stroke itself is the height minus one cap diameter.
|
||||
const half = (height - BAR_WIDTH) / 2;
|
||||
context.beginPath();
|
||||
if (half < 0.25) {
|
||||
context.arc(x, centerY, BAR_WIDTH / 2, 0, Math.PI * 2);
|
||||
context.fill();
|
||||
} else {
|
||||
context.moveTo(x, centerY - half);
|
||||
context.lineTo(x, centerY + half);
|
||||
context.stroke();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let frame = 0;
|
||||
let lastSampleAt = 0;
|
||||
const tick = (now: number) => {
|
||||
frame = requestAnimationFrame(tick);
|
||||
if (now - lastSampleAt < SAMPLE_INTERVAL_MS) {
|
||||
return;
|
||||
}
|
||||
lastSampleAt = now;
|
||||
bars.push(peakSinceSample);
|
||||
peakSinceSample = 0;
|
||||
draw();
|
||||
};
|
||||
frame = requestAnimationFrame(tick);
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
observer.disconnect();
|
||||
unsubscribe();
|
||||
};
|
||||
}, [subscribeLevel, barColor]);
|
||||
|
||||
return <canvas ref={canvasRef} className={className} aria-hidden="true" />;
|
||||
};
|
||||
@@ -2,16 +2,19 @@
|
||||
* Streaming dictation state machine.
|
||||
*
|
||||
* Status flow: idle -> recording -> uploading -> idle | failed.
|
||||
* While recording, mic PCM chunks stream to the server, which sends back live
|
||||
* partial transcripts. Confirm finalizes and resolves the full text; failed
|
||||
* dictations retain their audio segments so retry can replay them.
|
||||
* While recording, mic PCM chunks stream to the server, which transcribes them
|
||||
* segment by segment; confirm finalizes and resolves the full text. Nothing is
|
||||
* shown while recording — `partialTranscript` holds whatever the server has
|
||||
* transcribed so far and exists only so a failed dictation can be salvaged
|
||||
* instead of losing minutes of speech. Failed dictations also retain their
|
||||
* audio segments so retry can replay them.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { dictationClient, type DictationStartOptions } from '@/lib/dictation/dictation-client';
|
||||
import { DictationStreamSender } from '@/lib/dictation/dictation-stream-sender';
|
||||
import { useDictationAudioSource } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { useDictationAudioSource, type DictationLevelListener } from '@/lib/dictation/use-dictation-audio-source';
|
||||
import { useConfigStore } from '@/stores/useConfigStore';
|
||||
|
||||
export type DictationStatus = 'idle' | 'recording' | 'uploading' | 'failed';
|
||||
@@ -26,8 +29,10 @@ export interface UseDictationResult {
|
||||
status: DictationStatus;
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
/** Server-side transcript so far; recovery only, never shown while recording. */
|
||||
partialTranscript: string;
|
||||
volume: number;
|
||||
/** Subscribe to the normalized (0..1) mic level for the waveform. */
|
||||
subscribeLevel: (listener: DictationLevelListener) => () => void;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
errorReason: string | null;
|
||||
@@ -133,7 +138,8 @@ export function useDictation(options: UseDictationOptions = {}): UseDictationRes
|
||||
setPartialTranscript('');
|
||||
}, []);
|
||||
|
||||
// Live partial transcripts for the active dictation.
|
||||
// Transcripts of segments the server has already committed. Not rendered
|
||||
// while recording; kept so a failed dictation can be salvaged.
|
||||
useEffect(() => {
|
||||
return dictationClient.onPartial((dictationId, text) => {
|
||||
const activeDictationId = senderRef.current?.getDictationId();
|
||||
@@ -223,7 +229,8 @@ export function useDictation(options: UseDictationOptions = {}): UseDictationRes
|
||||
try {
|
||||
await audio.start();
|
||||
startDurationTracking();
|
||||
// Open the stream eagerly so partials start flowing immediately.
|
||||
// Open the stream eagerly so audio uploads while the user speaks
|
||||
// and only the tail is left to transcribe on stop.
|
||||
await senderRef.current?.restartStream().catch((err) => {
|
||||
// Non-fatal: segments buffer locally and finish() retries the
|
||||
// start, but surface the reason (e.g. model downloading) so
|
||||
@@ -394,7 +401,7 @@ export function useDictation(options: UseDictationOptions = {}): UseDictationRes
|
||||
isRecording: status === 'recording',
|
||||
isProcessing: status === 'uploading',
|
||||
partialTranscript,
|
||||
volume: audio.volume,
|
||||
subscribeLevel: audio.subscribeLevel,
|
||||
duration,
|
||||
error,
|
||||
errorReason,
|
||||
|
||||
@@ -4,20 +4,27 @@
|
||||
* Captures mono audio via getUserMedia, taps it with a ScriptProcessorNode
|
||||
* (universally supported, including iOS WKWebView), resamples Float32 to
|
||||
* 16 kHz PCM16LE, and emits ~1-second base64 chunks plus a normalized RMS
|
||||
* volume for the level meter.
|
||||
* level for the waveform.
|
||||
*
|
||||
* The level is delivered by subscription rather than React state: it updates
|
||||
* on every audio callback (~12 Hz), and routing that through state re-rendered
|
||||
* the whole dictation overlay at the same rate.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useRef } from 'react';
|
||||
|
||||
export interface DictationAudioSourceConfig {
|
||||
onPcmSegment: (base64Pcm: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
export type DictationLevelListener = (level: number) => void;
|
||||
|
||||
export interface DictationAudioSource {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
volume: number;
|
||||
/** Subscribe to the normalized (0..1) mic level. Returns an unsubscribe. */
|
||||
subscribeLevel: (listener: DictationLevelListener) => () => void;
|
||||
}
|
||||
|
||||
const OUTPUT_RATE = 16000;
|
||||
@@ -125,7 +132,18 @@ export const isDictationCaptureSupported = (): boolean => {
|
||||
};
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const [volume, setVolume] = useState(0);
|
||||
const levelListenersRef = useRef(new Set<DictationLevelListener>());
|
||||
const emitLevel = useCallback((level: number) => {
|
||||
for (const listener of levelListenersRef.current) {
|
||||
listener(level);
|
||||
}
|
||||
}, []);
|
||||
const subscribeLevel = useCallback((listener: DictationLevelListener) => {
|
||||
levelListenersRef.current.add(listener);
|
||||
return () => {
|
||||
levelListenersRef.current.delete(listener);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
@@ -196,7 +214,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
sumSquares += input[i] * input[i];
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
setVolume(Math.min(1, Math.max(0, rms * 2)));
|
||||
emitLevel(Math.min(1, Math.max(0, rms * 2)));
|
||||
|
||||
const next = resampleToPcm16(input, context.sampleRate, OUTPUT_RATE);
|
||||
graph.pending = concatInt16(graph.pending, next);
|
||||
@@ -227,12 +245,12 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
graphRef.current = emptyGraph();
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
}, []);
|
||||
}, [emitLevel]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
const graph = graphRef.current;
|
||||
graph.started = false;
|
||||
setVolume(0);
|
||||
emitLevel(0);
|
||||
|
||||
if (graph.processor) {
|
||||
try {
|
||||
@@ -272,7 +290,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
if (graphRef.current === graph) {
|
||||
graphRef.current = emptyGraph();
|
||||
}
|
||||
}, []);
|
||||
}, [emitLevel]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
@@ -294,8 +312,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
}
|
||||
},
|
||||
stop,
|
||||
volume,
|
||||
subscribeLevel,
|
||||
}),
|
||||
[start, stop, volume],
|
||||
[start, stop, subscribeLevel],
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user