Improve event stream resilience
This commit is contained in:
@@ -12,10 +12,20 @@ type Listener = (event: OpenChamberEvent) => void;
|
||||
|
||||
let eventSource: EventSource | null = null;
|
||||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let reconnectAttempt = 0;
|
||||
const listeners = new Set<Listener>();
|
||||
|
||||
const MAX_RECONNECT_DELAY_MS = 30_000;
|
||||
const HEARTBEAT_TIMEOUT_MS = 45_000;
|
||||
|
||||
const clearHeartbeatTimer = () => {
|
||||
if (!heartbeatTimer) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(heartbeatTimer);
|
||||
heartbeatTimer = null;
|
||||
};
|
||||
|
||||
const scheduleReconnect = () => {
|
||||
if (reconnectTimer || listeners.size === 0) {
|
||||
@@ -30,12 +40,24 @@ const scheduleReconnect = () => {
|
||||
};
|
||||
|
||||
const cleanupSource = () => {
|
||||
clearHeartbeatTimer();
|
||||
if (eventSource) {
|
||||
eventSource.close();
|
||||
}
|
||||
eventSource = null;
|
||||
};
|
||||
|
||||
const resetHeartbeatTimer = () => {
|
||||
clearHeartbeatTimer();
|
||||
if (listeners.size === 0) {
|
||||
return;
|
||||
}
|
||||
heartbeatTimer = setTimeout(() => {
|
||||
cleanupSource();
|
||||
scheduleReconnect();
|
||||
}, HEARTBEAT_TIMEOUT_MS);
|
||||
};
|
||||
|
||||
const parseEnvelope = (raw: string): { type: string; properties: unknown } | null => {
|
||||
if (!raw || raw.trim().length === 0) {
|
||||
return null;
|
||||
@@ -60,6 +82,10 @@ const dispatchFromEnvelope = (envelope: { type: string; properties: unknown }) =
|
||||
return;
|
||||
}
|
||||
|
||||
if (envelope.type === 'openchamber:heartbeat') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (envelope.type !== 'openchamber:scheduled-task-ran') {
|
||||
return;
|
||||
}
|
||||
@@ -93,10 +119,22 @@ const connect = () => {
|
||||
if (typeof window === 'undefined' || listeners.size === 0) {
|
||||
return;
|
||||
}
|
||||
if (typeof EventSource !== 'function') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventSource && eventSource.readyState !== EventSource.CLOSED) {
|
||||
return;
|
||||
}
|
||||
|
||||
cleanupSource();
|
||||
|
||||
const source = new EventSource('/api/openchamber/events');
|
||||
source.onopen = () => {
|
||||
resetHeartbeatTimer();
|
||||
};
|
||||
source.onmessage = (event) => {
|
||||
resetHeartbeatTimer();
|
||||
const envelope = parseEnvelope(event.data);
|
||||
if (!envelope) {
|
||||
return;
|
||||
|
||||
@@ -667,6 +667,85 @@ describe('createEventPipeline', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('passes the last websocket event id when falling back to SSE', async () => {
|
||||
installDomStubs();
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
const originalConsoleError = console.error;
|
||||
console.error = () => {};
|
||||
|
||||
let releaseStream;
|
||||
const hold = new Promise((resolve) => {
|
||||
releaseStream = resolve;
|
||||
});
|
||||
|
||||
const eventOptions = [];
|
||||
const received = [];
|
||||
const sdk = {
|
||||
global: {
|
||||
event: async (options) => {
|
||||
eventOptions.push(options);
|
||||
return {
|
||||
stream: (async function* () {
|
||||
yield {
|
||||
payload: {
|
||||
type: 'server.connected',
|
||||
properties: {},
|
||||
},
|
||||
};
|
||||
await hold;
|
||||
})(),
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const delivered = new Promise((resolve) => {
|
||||
const { cleanup } = createEventPipeline({
|
||||
sdk,
|
||||
transport: 'auto',
|
||||
reconnectDelayMs: 0,
|
||||
wsReadyTimeoutMs: 20,
|
||||
onEvent: (directory, payload) => {
|
||||
received.push({ directory, payload });
|
||||
if (payload.type !== 'server.connected') {
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
releaseStream();
|
||||
resolve();
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.resolve();
|
||||
|
||||
const firstSocket = FakeWebSocket.instances[0];
|
||||
firstSocket.emitOpen();
|
||||
firstSocket.emitMessage({ type: 'ready', scope: 'global' });
|
||||
firstSocket.emitMessage({
|
||||
type: 'event',
|
||||
eventId: 'evt-1',
|
||||
directory: '/tmp/project',
|
||||
payload: {
|
||||
type: 'session.status',
|
||||
properties: {
|
||||
sessionID: 'session-1',
|
||||
},
|
||||
},
|
||||
});
|
||||
firstSocket.emitClose();
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
await delivered;
|
||||
|
||||
expect(eventOptions[0]?.headers?.['Last-Event-ID']).toBe('evt-1');
|
||||
expect(received.some((entry) => entry.payload.type === 'server.connected')).toBe(true);
|
||||
} finally {
|
||||
console.error = originalConsoleError;
|
||||
}
|
||||
});
|
||||
|
||||
it('marks the pipeline disconnected on heartbeat timeout and recovers on the next websocket connect', async () => {
|
||||
installDomStubs();
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
|
||||
@@ -339,6 +339,7 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
const runSseAttempt = async (signal: AbortSignal) => {
|
||||
const events = await sdk.global.event({
|
||||
signal,
|
||||
...(lastEventId && lastEventId.length > 0 ? { headers: { "Last-Event-ID": lastEventId } } : {}),
|
||||
onSseError: (error: unknown) => {
|
||||
if (isAbortError(error)) return
|
||||
if (streamErrorLogged) return
|
||||
|
||||
Reference in New Issue
Block a user