fix(server): prevent streaming hang during long agent sessions (#1088)
* fix(server): increase WS buffer/replay limits and add backpressure warning During long-running agent sessions (e.g. ultrawork loops with many tool calls), the browser WebSocket client can briefly fall behind the server. When the outbound buffer exceeds the limit, the server force-disconnects with close code 1013, and the small replay buffer (512 events) is insufficient to recover all missed events — leaving the UI permanently stalled. Changes: - Raise MESSAGE_STREAM_WS_MAX_BUFFERED_BYTES from 4 MB to 16 MB to tolerate larger bursts without disconnecting - Add MESSAGE_STREAM_WS_BACKPRESSURE_WARN_BYTES (12 MB) threshold that sends a one-shot "backpressure" frame to the client before the hard disconnect, giving it a chance to shed low-priority updates - Raise MESSAGE_STREAM_GLOBAL_REPLAY_LIMIT from 512 to 2048 so more events survive brief reconnection gaps - Add tests for the backpressure warning behavior (emit, dedup, reset) * fix(ui): batch event flushes under backpressure --------- Co-authored-by: Bohdan Triapitsyn <artmore@protonmail.com>
This commit is contained in:
committed by
GitHub
co-authored by
Bohdan Triapitsyn
parent
03c9065c90
commit
1991736ebf
@@ -20,6 +20,8 @@ export type QueuedEvent = {
|
||||
export type FlushHandler = (events: QueuedEvent[]) => void
|
||||
|
||||
const FLUSH_FRAME_MS = 33
|
||||
const BACKPRESSURE_FLUSH_FRAME_MS = 200
|
||||
const BACKPRESSURE_MODE_MS = 10_000
|
||||
const STREAM_YIELD_MS = 8
|
||||
const DEFAULT_RECONNECT_DELAY_MS = 250
|
||||
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 30_000
|
||||
@@ -44,7 +46,7 @@ export type EventPipelineInput = {
|
||||
}
|
||||
|
||||
type MessageStreamWsFrame = {
|
||||
type: "ready" | "event" | "error"
|
||||
type: "ready" | "event" | "error" | "backpressure"
|
||||
payload?: unknown
|
||||
eventId?: string
|
||||
directory?: string
|
||||
@@ -254,7 +256,8 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
const d = getOrCreateDir(directory)
|
||||
if (d.timer) return
|
||||
const elapsed = Date.now() - d.last
|
||||
d.timer = setTimeout(() => flushDir(directory), Math.max(0, FLUSH_FRAME_MS - elapsed))
|
||||
const flushFrameMs = Date.now() < backpressureUntil ? BACKPRESSURE_FLUSH_FRAME_MS : FLUSH_FRAME_MS
|
||||
d.timer = setTimeout(() => flushDir(directory), Math.max(0, flushFrameMs - elapsed))
|
||||
}
|
||||
|
||||
const wait = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
|
||||
@@ -269,6 +272,7 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
let activeTransport: "ws" | "sse" = transport === "ws" ? "ws" : "sse"
|
||||
let attemptAbortReason: AttemptAbortReason = null
|
||||
let consecutiveFailures = 0
|
||||
let backpressureUntil = 0
|
||||
|
||||
const notifyDisconnected = (reason: string) => {
|
||||
if (disconnected) {
|
||||
@@ -489,6 +493,11 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
return
|
||||
}
|
||||
|
||||
if (frame.type === "backpressure") {
|
||||
backpressureUntil = Date.now() + BACKPRESSURE_MODE_MS
|
||||
return
|
||||
}
|
||||
|
||||
if (frame.type !== "event") {
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user