Files
openchamber/packages/web/server/lib/event-stream/protocol.js
T
pasta-paulandBohdan Triapitsyn 1991736ebf fix(server): prevent streaming hang during long agent sessions (#1088)
* fix(server): increase WS buffer/replay limits and add backpressure warning

During long-running agent sessions (e.g. ultrawork loops with many tool
calls), the browser WebSocket client can briefly fall behind the server.
When the outbound buffer exceeds the limit, the server force-disconnects
with close code 1013, and the small replay buffer (512 events) is
insufficient to recover all missed events — leaving the UI permanently
stalled.

Changes:
- Raise MESSAGE_STREAM_WS_MAX_BUFFERED_BYTES from 4 MB to 16 MB to
  tolerate larger bursts without disconnecting
- Add MESSAGE_STREAM_WS_BACKPRESSURE_WARN_BYTES (12 MB) threshold that
  sends a one-shot "backpressure" frame to the client before the hard
  disconnect, giving it a chance to shed low-priority updates
- Raise MESSAGE_STREAM_GLOBAL_REPLAY_LIMIT from 512 to 2048 so more
  events survive brief reconnection gaps
- Add tests for the backpressure warning behavior (emit, dedup, reset)

* fix(ui): batch event flushes under backpressure

---------
Co-authored-by: Bohdan Triapitsyn <artmore@protonmail.com>
2026-05-01 13:25:19 +03:00

132 lines
4.0 KiB
JavaScript

export const MESSAGE_STREAM_GLOBAL_WS_PATH = '/api/global/event/ws';
export const MESSAGE_STREAM_DIRECTORY_WS_PATH = '/api/event/ws';
export const MESSAGE_STREAM_WS_HEARTBEAT_INTERVAL_MS = 15 * 1000;
// Per-client pending outbound WS buffer, not a payload or stream-size limit.
// Healthy clients stay near 0; this only trips when a client is far behind.
// Raised from 4 MB → 16 MB to tolerate bursts during long agent sessions
// (e.g. ultrawork / multi-tool loops) where the browser briefly falls behind.
export const MESSAGE_STREAM_WS_MAX_BUFFERED_BYTES = 16 * 1024 * 1024;
// Threshold at which we emit a backpressure warning frame so the client can
// proactively start shedding low-priority updates before the hard disconnect.
export const MESSAGE_STREAM_WS_BACKPRESSURE_WARN_BYTES = 12 * 1024 * 1024;
export function parseSseEventEnvelope(block) {
if (!block || typeof block !== 'string') {
return null;
}
const eventId = block
.split('\n')
.find((line) => line.startsWith('id:'))
?.slice(3)
.trim() || null;
const dataLines = block
.split('\n')
.filter((line) => line.startsWith('data:'))
.map((line) => line.slice(5).replace(/^\s/, ''));
if (dataLines.length === 0) {
return null;
}
const payloadText = dataLines.join('\n').trim();
if (!payloadText) {
return null;
}
try {
const parsed = JSON.parse(payloadText);
if (
parsed &&
typeof parsed === 'object' &&
typeof parsed.payload === 'object' &&
parsed.payload !== null
) {
return {
eventId,
directory: typeof parsed.directory === 'string' && parsed.directory.length > 0 ? parsed.directory : null,
payload: parsed.payload,
};
}
const directory =
typeof parsed?.directory === 'string' && parsed.directory.length > 0
? parsed.directory
: typeof parsed?.properties?.directory === 'string' && parsed.properties.directory.length > 0
? parsed.properties.directory
: null;
return {
eventId,
directory,
payload: parsed,
};
} catch {
return null;
}
}
export function sendMessageStreamWsFrame(socket, payload) {
if (!socket || socket.readyState !== 1) {
return false;
}
const buffered = typeof socket.bufferedAmount === 'number' ? socket.bufferedAmount : 0;
if (buffered > MESSAGE_STREAM_WS_MAX_BUFFERED_BYTES) {
try {
socket.close(1013, 'Message stream client is too slow');
} catch {
}
return false;
}
try {
socket.send(JSON.stringify(payload));
const bufferedAfter = typeof socket.bufferedAmount === 'number' ? socket.bufferedAmount : 0;
if (bufferedAfter > MESSAGE_STREAM_WS_MAX_BUFFERED_BYTES) {
try {
socket.close(1013, 'Message stream client is too slow');
} catch {
}
return false;
}
// Emit a one-shot backpressure warning when the buffer is building up.
// The flag prevents sending repeated warnings that would themselves
// increase the buffer. It resets once the buffer drains below the
// threshold.
if (bufferedAfter > MESSAGE_STREAM_WS_BACKPRESSURE_WARN_BYTES) {
if (!socket._ocBackpressureWarned) {
socket._ocBackpressureWarned = true;
try {
socket.send(JSON.stringify({
type: 'backpressure',
bufferedBytes: bufferedAfter,
maxBytes: MESSAGE_STREAM_WS_MAX_BUFFERED_BYTES,
}));
} catch {
// Best-effort warning — ignore send failures.
}
}
} else if (socket._ocBackpressureWarned) {
socket._ocBackpressureWarned = false;
}
return true;
} catch {
return false;
}
}
export function sendMessageStreamWsEvent(socket, payload, options = {}) {
return sendMessageStreamWsFrame(socket, {
type: 'event',
payload,
...(typeof options.eventId === 'string' && options.eventId.length > 0 ? { eventId: options.eventId } : {}),
...(typeof options.directory === 'string' && options.directory.length > 0 ? { directory: options.directory } : {}),
});
}