fix: improve event stream reconnect reliability
Recover stalled event streams without dropping the session Wait briefly for reconnection before showing connection lost errors Persist Electron server logs for easier disconnect debugging
This commit is contained in:
@@ -42,6 +42,14 @@ log.transports.file.maxSize = 5 * 1024 * 1024;
|
||||
log.transports.file.level = 'info';
|
||||
log.transports.console.level = isDev ? 'debug' : 'warn';
|
||||
|
||||
// The in-process web server runs in this same Node process and uses plain
|
||||
// `console.log/warn/error`. Without piping console through electron-log,
|
||||
// that output never lands in ~/Library/Logs/OpenChamber/main.log and we
|
||||
// can't diagnose issues (e.g. OpenCode lifecycle, SSE disconnects) after
|
||||
// the fact. Route all console calls through electron-log so server-side
|
||||
// diagnostics are persisted.
|
||||
Object.assign(console, log.functions);
|
||||
|
||||
const LOG_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
try {
|
||||
const logPath = log.transports.file.getFile().path;
|
||||
|
||||
@@ -1519,11 +1519,7 @@ const ChatInputComponent: React.FC<ChatInputProps> = ({ onOpenSettings, scrollTo
|
||||
}
|
||||
else if (commandName === 'compact' && currentSessionId) {
|
||||
try {
|
||||
if (!useConfigStore.getState().isConnected) {
|
||||
const reason = useConfigStore.getState().lastDisconnectReason;
|
||||
const suffix = reason ? ` (${reason})` : " (never connected)";
|
||||
throw new Error(`Connection lost${suffix}. Please wait for reconnection.`);
|
||||
}
|
||||
await sessionActions.waitForConnectionOrThrow();
|
||||
const { opencodeClient } = await import('@/lib/opencode/client');
|
||||
const sdk = opencodeClient.getSdkClient();
|
||||
const configState = useConfigStore.getState();
|
||||
|
||||
@@ -468,6 +468,8 @@ interface ConfigStore {
|
||||
agentModelSelections: { [agentName: string]: { providerId: string; modelId: string } };
|
||||
defaultProviders: { [key: string]: string };
|
||||
isConnected: boolean;
|
||||
hasEverConnected: boolean;
|
||||
connectionPhase: "connecting" | "connected" | "reconnecting";
|
||||
lastDisconnectReason: string | null;
|
||||
isInitialized: boolean;
|
||||
modelsMetadata: Map<string, ModelMetadata>;
|
||||
@@ -589,6 +591,8 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
agentModelSelections: {},
|
||||
defaultProviders: {},
|
||||
isConnected: false,
|
||||
hasEverConnected: false,
|
||||
connectionPhase: "connecting",
|
||||
lastDisconnectReason: null,
|
||||
isInitialized: false,
|
||||
modelsMetadata: new Map<string, ModelMetadata>(),
|
||||
@@ -1890,7 +1894,14 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
while (attempt < maxAttempts) {
|
||||
try {
|
||||
const isHealthy = await opencodeClient.checkHealth();
|
||||
set({ isConnected: isHealthy });
|
||||
const hasEverConnected = get().hasEverConnected;
|
||||
set(isHealthy
|
||||
? { isConnected: true, hasEverConnected: true, connectionPhase: "connected" }
|
||||
: {
|
||||
isConnected: false,
|
||||
connectionPhase: hasEverConnected ? "reconnecting" : "connecting",
|
||||
lastDisconnectReason: 'health_check_unhealthy',
|
||||
});
|
||||
return isHealthy;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
@@ -1903,7 +1914,11 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
if (lastError) {
|
||||
console.warn("[ConfigStore] Failed to reach OpenCode after retrying:", lastError);
|
||||
}
|
||||
set({ isConnected: false });
|
||||
set({
|
||||
isConnected: false,
|
||||
connectionPhase: get().hasEverConnected ? "reconnecting" : "connecting",
|
||||
lastDisconnectReason: 'health_check_failed',
|
||||
});
|
||||
return false;
|
||||
},
|
||||
|
||||
@@ -1917,7 +1932,11 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
|
||||
if (!isConnected) {
|
||||
if (debug) console.log("Server not connected");
|
||||
set({ isConnected: false });
|
||||
// checkConnection already set lastDisconnectReason; do not overwrite.
|
||||
set({
|
||||
isConnected: false,
|
||||
connectionPhase: get().hasEverConnected ? "reconnecting" : "connecting",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1930,11 +1949,16 @@ export const useConfigStore = create<ConfigStore>()(
|
||||
if (debug) console.log("Loading agents...");
|
||||
await get().loadAgents();
|
||||
|
||||
set({ isInitialized: true, isConnected: true });
|
||||
set({ isInitialized: true, isConnected: true, hasEverConnected: true, connectionPhase: "connected" });
|
||||
if (debug) console.log("App initialized successfully");
|
||||
} catch (error) {
|
||||
console.error("Failed to initialize app:", error);
|
||||
set({ isInitialized: false, isConnected: false });
|
||||
set({
|
||||
isInitialized: false,
|
||||
isConnected: false,
|
||||
connectionPhase: get().hasEverConnected ? "reconnecting" : "connecting",
|
||||
lastDisconnectReason: 'init_error',
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -666,6 +666,62 @@ describe('createEventPipeline', () => {
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('marks the pipeline disconnected on heartbeat timeout and recovers on the next websocket connect', async () => {
|
||||
installDomStubs();
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
|
||||
const disconnectReasons = [];
|
||||
let reconnectCount = 0;
|
||||
|
||||
const sdk = {
|
||||
global: {
|
||||
event: async () => {
|
||||
throw new Error('SSE should not be used in ws mode');
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const recovered = new Promise((resolve) => {
|
||||
const { cleanup } = createEventPipeline({
|
||||
sdk,
|
||||
transport: 'ws',
|
||||
heartbeatTimeoutMs: 20,
|
||||
reconnectDelayMs: 0,
|
||||
wsReadyTimeoutMs: 20,
|
||||
onEvent: () => {},
|
||||
onDisconnect: (reason) => {
|
||||
disconnectReasons.push(reason);
|
||||
},
|
||||
onReconnect: () => {
|
||||
reconnectCount += 1;
|
||||
if (reconnectCount === 2) {
|
||||
cleanup();
|
||||
resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
|
||||
const firstSocket = FakeWebSocket.instances[0];
|
||||
firstSocket.emitOpen();
|
||||
firstSocket.emitMessage({ type: 'ready', scope: 'global' });
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 35));
|
||||
|
||||
const secondSocket = FakeWebSocket.instances[1];
|
||||
expect(secondSocket).toBeDefined();
|
||||
|
||||
secondSocket.emitOpen();
|
||||
secondSocket.emitMessage({ type: 'ready', scope: 'global' });
|
||||
|
||||
await recovered;
|
||||
|
||||
expect(disconnectReasons).toEqual(['ws_heartbeat_timeout']);
|
||||
expect(reconnectCount).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -21,10 +21,10 @@ export type FlushHandler = (events: QueuedEvent[]) => void
|
||||
|
||||
const FLUSH_FRAME_MS = 33
|
||||
const STREAM_YIELD_MS = 8
|
||||
const RECONNECT_DELAY_MS = 250
|
||||
const HEARTBEAT_TIMEOUT_MS = 15_000
|
||||
const DEFAULT_RECONNECT_DELAY_MS = 250
|
||||
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 30_000
|
||||
const WS_FALLBACK_WINDOW_MS = 60_000
|
||||
const WS_READY_TIMEOUT_MS = 2_000
|
||||
const DEFAULT_WS_READY_TIMEOUT_MS = 2_000
|
||||
const ABSOLUTE_URL_PATTERN = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//
|
||||
|
||||
export type EventPipelineInput = {
|
||||
@@ -38,6 +38,9 @@ export type EventPipelineInput = {
|
||||
/** Called when transport switches (e.g. WS timeout → SSE fallback) without actual disconnection. */
|
||||
onTransportSwitch?: () => void
|
||||
transport?: "auto" | "ws" | "sse"
|
||||
heartbeatTimeoutMs?: number
|
||||
reconnectDelayMs?: number
|
||||
wsReadyTimeoutMs?: number
|
||||
}
|
||||
|
||||
type MessageStreamWsFrame = {
|
||||
@@ -145,8 +148,25 @@ type DirectoryQueue = {
|
||||
last: number
|
||||
}
|
||||
|
||||
type AttemptAbortReason =
|
||||
| "pipeline_stopped"
|
||||
| "ws_heartbeat_timeout"
|
||||
| "sse_heartbeat_timeout"
|
||||
| null
|
||||
|
||||
export function createEventPipeline(input: EventPipelineInput) {
|
||||
const { sdk, onEvent, onReconnect, onDisconnect, onTransportSwitch, routeDirectory, transport = "auto" } = input
|
||||
const {
|
||||
sdk,
|
||||
onEvent,
|
||||
onReconnect,
|
||||
onDisconnect,
|
||||
onTransportSwitch,
|
||||
routeDirectory,
|
||||
transport = "auto",
|
||||
heartbeatTimeoutMs = DEFAULT_HEARTBEAT_TIMEOUT_MS,
|
||||
reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS,
|
||||
wsReadyTimeoutMs = DEFAULT_WS_READY_TIMEOUT_MS,
|
||||
} = input
|
||||
const abort = new AbortController()
|
||||
let disconnected = false
|
||||
let lastEventId: string | undefined
|
||||
@@ -244,6 +264,16 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
let attempt: AbortController | undefined
|
||||
let lastEventAt = Date.now()
|
||||
let heartbeat: ReturnType<typeof setTimeout> | undefined
|
||||
let activeTransport: "ws" | "sse" = transport === "ws" ? "ws" : "sse"
|
||||
let attemptAbortReason: AttemptAbortReason = null
|
||||
|
||||
const notifyDisconnected = (reason: string) => {
|
||||
if (disconnected) {
|
||||
return
|
||||
}
|
||||
disconnected = true
|
||||
onDisconnect?.(reason)
|
||||
}
|
||||
|
||||
const markConnected = () => {
|
||||
disconnected = false
|
||||
@@ -295,8 +325,9 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
lastEventAt = Date.now()
|
||||
if (heartbeat) clearTimeout(heartbeat)
|
||||
heartbeat = setTimeout(() => {
|
||||
attemptAbortReason = `${activeTransport}_heartbeat_timeout`
|
||||
attempt?.abort()
|
||||
}, HEARTBEAT_TIMEOUT_MS)
|
||||
}, heartbeatTimeoutMs)
|
||||
}
|
||||
|
||||
const clearHeartbeat = () => {
|
||||
@@ -359,7 +390,7 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}, WS_READY_TIMEOUT_MS)
|
||||
}, wsReadyTimeoutMs)
|
||||
|
||||
const cleanup = () => {
|
||||
if (readyTimer) {
|
||||
@@ -499,9 +530,12 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
while (!abort.signal.aborted) {
|
||||
attempt = new AbortController()
|
||||
lastEventAt = Date.now()
|
||||
let retryDelayMs = RECONNECT_DELAY_MS
|
||||
attemptAbortReason = null
|
||||
let retryDelayMs = reconnectDelayMs
|
||||
const currentTransport = resolveTransport()
|
||||
activeTransport = currentTransport
|
||||
const onAbort = () => {
|
||||
attemptAbortReason = "pipeline_stopped"
|
||||
attempt?.abort()
|
||||
}
|
||||
abort.signal.addEventListener("abort", onAbort)
|
||||
@@ -531,21 +565,18 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
// update connection state (e.g. set isConnected = false).
|
||||
// Guard: only fire once per disconnection cycle to avoid repeated
|
||||
// setState calls on every failed retry attempt.
|
||||
if (!disconnected) {
|
||||
disconnected = true
|
||||
const taggedReason = typeof error === "object" && error !== null
|
||||
? (error as { reason?: unknown }).reason
|
||||
: undefined
|
||||
const message = typeof error === "object" && error !== null
|
||||
? (error as { message?: unknown }).message
|
||||
: undefined
|
||||
const reason = typeof taggedReason === "string" && taggedReason.length > 0
|
||||
? taggedReason
|
||||
: typeof message === "string" && message.length > 0
|
||||
? `${currentTransport}_error:${message.slice(0, 80)}`
|
||||
: `${currentTransport}_error:unknown`
|
||||
onDisconnect?.(reason)
|
||||
}
|
||||
const taggedReason = typeof error === "object" && error !== null
|
||||
? (error as { reason?: unknown }).reason
|
||||
: undefined
|
||||
const message = typeof error === "object" && error !== null
|
||||
? (error as { message?: unknown }).message
|
||||
: undefined
|
||||
const reason = typeof taggedReason === "string" && taggedReason.length > 0
|
||||
? taggedReason
|
||||
: typeof message === "string" && message.length > 0
|
||||
? `${currentTransport}_error:${message.slice(0, 80)}`
|
||||
: `${currentTransport}_error:unknown`
|
||||
notifyDisconnected(reason)
|
||||
}
|
||||
} finally {
|
||||
abort.signal.removeEventListener("abort", onAbort)
|
||||
@@ -554,6 +585,11 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
}
|
||||
|
||||
if (abort.signal.aborted) return
|
||||
if (attemptAbortReason && attemptAbortReason !== "pipeline_stopped") {
|
||||
notifyDisconnected(attemptAbortReason)
|
||||
retryDelayMs = 0
|
||||
attemptAbortReason = null
|
||||
}
|
||||
if (retryDelayMs > 0) {
|
||||
await wait(retryDelayMs)
|
||||
}
|
||||
@@ -563,7 +599,7 @@ export function createEventPipeline(input: EventPipelineInput) {
|
||||
const onVisibility = () => {
|
||||
if (typeof document === "undefined") return
|
||||
if (document.visibilityState !== "visible") return
|
||||
if (Date.now() - lastEventAt < HEARTBEAT_TIMEOUT_MS) return
|
||||
if (Date.now() - lastEventAt < heartbeatTimeoutMs) return
|
||||
attempt?.abort()
|
||||
}
|
||||
|
||||
|
||||
@@ -55,11 +55,30 @@ function dir() {
|
||||
}
|
||||
|
||||
function connectionLostError(): Error {
|
||||
const reason = useConfigStore.getState().lastDisconnectReason
|
||||
const suffix = reason ? ` (${reason})` : " (never connected)"
|
||||
const { hasEverConnected, lastDisconnectReason } = useConfigStore.getState()
|
||||
const suffix = lastDisconnectReason
|
||||
? ` (${lastDisconnectReason})`
|
||||
: hasEverConnected
|
||||
? ""
|
||||
: " (never connected)"
|
||||
return new Error(`Connection lost${suffix}. Please wait for reconnection.`)
|
||||
}
|
||||
|
||||
// Wait briefly for the pipeline to re-establish connection before failing a
|
||||
// send. Transient reconnects (heartbeat race, WS→SSE fallback, brief network
|
||||
// blip) otherwise surface as a hard "Connection lost" toast even though the
|
||||
// pipeline recovers within a second. Poll isConnected at 100ms intervals.
|
||||
const CONNECTION_GRACE_MS = 2000
|
||||
export async function waitForConnectionOrThrow(): Promise<void> {
|
||||
if (useConfigStore.getState().isConnected) return
|
||||
const deadline = Date.now() + CONNECTION_GRACE_MS
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||
if (useConfigStore.getState().isConnected) return
|
||||
}
|
||||
throw connectionLostError()
|
||||
}
|
||||
|
||||
function getSessionDirectory(sessionId: string): string | undefined {
|
||||
return useSessionUIStore.getState().getDirectoryForSession(sessionId) || dir()
|
||||
}
|
||||
@@ -332,9 +351,7 @@ export async function optimisticSend(input: {
|
||||
throw new Error("Optimistic refs not set — is useSync() mounted?")
|
||||
}
|
||||
|
||||
if (!useConfigStore.getState().isConnected) {
|
||||
throw connectionLostError()
|
||||
}
|
||||
await waitForConnectionOrThrow()
|
||||
|
||||
const store = dirStore()
|
||||
const messageID = ascendingId("msg")
|
||||
@@ -419,9 +436,7 @@ export async function respondToPermission(
|
||||
requestId: string,
|
||||
response: "once" | "always" | "reject",
|
||||
): Promise<void> {
|
||||
if (!useConfigStore.getState().isConnected) {
|
||||
throw connectionLostError()
|
||||
}
|
||||
await waitForConnectionOrThrow()
|
||||
const result = await getRequestReplyClient("permission", sessionId, requestId).permission.reply({
|
||||
requestID: requestId,
|
||||
reply: response,
|
||||
@@ -435,9 +450,7 @@ export async function dismissPermission(
|
||||
sessionId: string,
|
||||
requestId: string,
|
||||
): Promise<void> {
|
||||
if (!useConfigStore.getState().isConnected) {
|
||||
throw connectionLostError()
|
||||
}
|
||||
await waitForConnectionOrThrow()
|
||||
const result = await getRequestReplyClient("permission", sessionId, requestId).permission.reply({
|
||||
requestID: requestId,
|
||||
reply: "reject",
|
||||
@@ -456,9 +469,7 @@ export async function respondToQuestion(
|
||||
requestId: string,
|
||||
answers: string[] | string[][],
|
||||
): Promise<void> {
|
||||
if (!useConfigStore.getState().isConnected) {
|
||||
throw connectionLostError()
|
||||
}
|
||||
await waitForConnectionOrThrow()
|
||||
const result = await getRequestReplyClient("question", sessionId, requestId).question.reply({
|
||||
requestID: requestId,
|
||||
answers: answers as Array<Array<string>>,
|
||||
@@ -472,9 +483,7 @@ export async function rejectQuestion(
|
||||
sessionId: string,
|
||||
requestId: string,
|
||||
): Promise<void> {
|
||||
if (!useConfigStore.getState().isConnected) {
|
||||
throw connectionLostError()
|
||||
}
|
||||
await waitForConnectionOrThrow()
|
||||
const result = await getRequestReplyClient("question", sessionId, requestId).question.reject({
|
||||
requestID: requestId,
|
||||
})
|
||||
|
||||
@@ -1376,7 +1376,11 @@ export function SyncProvider(props: {
|
||||
handleEvent(directory, payload, childStores, routingIndex)
|
||||
},
|
||||
onReconnect: () => {
|
||||
useConfigStore.setState({ isConnected: true, lastDisconnectReason: null })
|
||||
useConfigStore.setState({
|
||||
isConnected: true,
|
||||
hasEverConnected: true,
|
||||
connectionPhase: "connected",
|
||||
})
|
||||
for (const [dir, store] of childStores.children) {
|
||||
if (reconnectResyncing.has(dir)) continue
|
||||
if (getReconnectCandidateSessionIds(store.getState()).length === 0) continue
|
||||
@@ -1392,13 +1396,22 @@ export function SyncProvider(props: {
|
||||
}
|
||||
},
|
||||
onDisconnect: (reason) => {
|
||||
useConfigStore.setState({ isConnected: false, lastDisconnectReason: reason })
|
||||
const { hasEverConnected } = useConfigStore.getState()
|
||||
useConfigStore.setState({
|
||||
isConnected: false,
|
||||
connectionPhase: hasEverConnected ? "reconnecting" : "connecting",
|
||||
lastDisconnectReason: reason,
|
||||
})
|
||||
},
|
||||
onTransportSwitch: () => {
|
||||
// Transport switched (e.g. WS timeout → SSE fallback) without
|
||||
// actual disconnection. No events lost — just update connection
|
||||
// state without triggering a full directory resync.
|
||||
useConfigStore.setState({ isConnected: true })
|
||||
useConfigStore.setState({
|
||||
isConnected: true,
|
||||
hasEverConnected: true,
|
||||
connectionPhase: "connected",
|
||||
})
|
||||
},
|
||||
})
|
||||
return cleanup
|
||||
|
||||
@@ -28,7 +28,9 @@ This module contains the OpenChamber message-stream WebSocket protocol and runti
|
||||
- Browser clients connect to the WS endpoints above.
|
||||
- OpenChamber still fetches OpenCode upstream event streams over SSE.
|
||||
- Each WS connection proxies one upstream SSE stream.
|
||||
- Global synthetic events such as `openchamber:session-status`, `openchamber:session-activity`, `openchamber:notification`, and `openchamber:heartbeat` are preserved on the WS path.
|
||||
- If an upstream SSE stream stalls after the browser WS is already ready, the runtime aborts that upstream fetch and reconnects upstream with `Last-Event-ID`, keeping the browser WS alive when recovery is fast.
|
||||
- Health checks are reserved for initial upstream connect failures and explicit upstream-unavailable responses, not for ordinary stall recovery on an already-established stream.
|
||||
- Global synthetic events such as `openchamber:session-status`, `openchamber:session-activity`, `openchamber:notification`, and `openchamber:heartbeat` are preserved on the WS path, but heartbeat frames are emitted only while an upstream SSE stream is actively attached.
|
||||
- Global UI broadcasts are fan-out capable across both SSE and WS clients.
|
||||
|
||||
## Notes for contributors
|
||||
|
||||
@@ -10,6 +10,9 @@ import {
|
||||
sendMessageStreamWsFrame,
|
||||
} from './protocol.js';
|
||||
|
||||
const MESSAGE_STREAM_UPSTREAM_STALL_TIMEOUT_MS = 20_000;
|
||||
const MESSAGE_STREAM_UPSTREAM_RECONNECT_DELAY_MS = 250;
|
||||
|
||||
function shouldTriggerUpstreamHealthCheck(upstream) {
|
||||
if (!upstream) {
|
||||
return true;
|
||||
@@ -67,6 +70,9 @@ export function createMessageStreamWsRuntime({
|
||||
processForwardedEventPayload,
|
||||
wsClients,
|
||||
triggerHealthCheck,
|
||||
heartbeatIntervalMs = MESSAGE_STREAM_WS_HEARTBEAT_INTERVAL_MS,
|
||||
upstreamStallTimeoutMs = MESSAGE_STREAM_UPSTREAM_STALL_TIMEOUT_MS,
|
||||
upstreamReconnectDelayMs = MESSAGE_STREAM_UPSTREAM_RECONNECT_DELAY_MS,
|
||||
fetchImpl = fetch,
|
||||
}) {
|
||||
const wsServer = new WebSocketServer({
|
||||
@@ -82,13 +88,22 @@ export function createMessageStreamWsRuntime({
|
||||
const requestedDirectory = requestUrl.searchParams.get('directory')?.trim() || '';
|
||||
|
||||
const controller = new AbortController();
|
||||
let currentUpstreamAbortController = null;
|
||||
let upstreamConnected = false;
|
||||
let streamReady = false;
|
||||
let lastEventId = requestedLastEventId;
|
||||
const cleanup = () => {
|
||||
if (!controller.signal.aborted) {
|
||||
controller.abort();
|
||||
}
|
||||
if (currentUpstreamAbortController && !currentUpstreamAbortController.signal.aborted) {
|
||||
currentUpstreamAbortController.abort();
|
||||
}
|
||||
wsClients.delete(socket);
|
||||
};
|
||||
|
||||
const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const pingInterval = setInterval(() => {
|
||||
if (socket.readyState !== 1) {
|
||||
return;
|
||||
@@ -98,15 +113,20 @@ export function createMessageStreamWsRuntime({
|
||||
socket.ping();
|
||||
} catch {
|
||||
}
|
||||
}, MESSAGE_STREAM_WS_HEARTBEAT_INTERVAL_MS);
|
||||
}, heartbeatIntervalMs);
|
||||
|
||||
const heartbeatInterval = setInterval(() => {
|
||||
if (!upstreamConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
sendMessageStreamWsEvent(socket, { type: 'openchamber:heartbeat', timestamp: Date.now() }, { directory: 'global' });
|
||||
}, MESSAGE_STREAM_WS_HEARTBEAT_INTERVAL_MS);
|
||||
}, heartbeatIntervalMs);
|
||||
|
||||
socket.on('close', () => {
|
||||
clearInterval(pingInterval);
|
||||
clearInterval(heartbeatInterval);
|
||||
upstreamConnected = false;
|
||||
cleanup();
|
||||
});
|
||||
|
||||
@@ -115,73 +135,6 @@ export function createMessageStreamWsRuntime({
|
||||
});
|
||||
|
||||
const run = async () => {
|
||||
let targetUrl;
|
||||
try {
|
||||
targetUrl = new URL(buildOpenCodeUrl(isGlobalStream ? '/global/event' : '/event', ''));
|
||||
} catch {
|
||||
sendMessageStreamWsFrame(socket, { type: 'error', message: 'OpenCode service unavailable' });
|
||||
socket.close(1011, 'OpenCode service unavailable');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isGlobalStream && requestedDirectory) {
|
||||
targetUrl.searchParams.set('directory', requestedDirectory);
|
||||
}
|
||||
|
||||
const headers = {
|
||||
Accept: 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
...getOpenCodeAuthHeaders(),
|
||||
};
|
||||
|
||||
if (requestedLastEventId) {
|
||||
headers['Last-Event-ID'] = requestedLastEventId;
|
||||
}
|
||||
|
||||
let upstream;
|
||||
try {
|
||||
upstream = await fetchImpl(targetUrl.toString(), {
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch {
|
||||
if (!controller.signal.aborted) {
|
||||
sendMessageStreamWsFrame(socket, { type: 'error', message: 'Failed to connect to OpenCode event stream' });
|
||||
socket.close(1011, 'Failed to connect to OpenCode event stream');
|
||||
// Trigger immediate health check so the server detects and
|
||||
// restarts a dead OpenCode process without waiting for the next
|
||||
// periodic interval (up to 15 s).
|
||||
triggerHealthCheck?.();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!upstream.ok || !upstream.body) {
|
||||
sendMessageStreamWsFrame(socket, {
|
||||
type: 'error',
|
||||
message: `OpenCode event stream unavailable (${upstream.status})`,
|
||||
});
|
||||
socket.close(1011, 'OpenCode event stream unavailable');
|
||||
if (shouldTriggerUpstreamHealthCheck(upstream)) {
|
||||
triggerHealthCheck?.();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
sendMessageStreamWsFrame(socket, {
|
||||
type: 'ready',
|
||||
scope: isGlobalStream ? 'global' : 'directory',
|
||||
});
|
||||
|
||||
if (isGlobalStream) {
|
||||
wsClients.add(socket);
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
const reader = upstream.body.getReader();
|
||||
let buffer = '';
|
||||
|
||||
const forwardBlock = (block) => {
|
||||
if (!block) {
|
||||
return;
|
||||
@@ -197,6 +150,10 @@ export function createMessageStreamWsRuntime({
|
||||
? (typeof envelope?.directory === 'string' && envelope.directory.length > 0 ? envelope.directory : 'global')
|
||||
: (requestedDirectory || envelope?.directory || 'global');
|
||||
|
||||
if (typeof envelope?.eventId === 'string' && envelope.eventId.length > 0) {
|
||||
lastEventId = envelope.eventId;
|
||||
}
|
||||
|
||||
sendMessageStreamWsEvent(socket, payload, {
|
||||
directory,
|
||||
eventId: typeof envelope?.eventId === 'string' && envelope.eventId.length > 0 ? envelope.eventId : undefined,
|
||||
@@ -208,25 +165,148 @@ export function createMessageStreamWsRuntime({
|
||||
};
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
while (!controller.signal.aborted) {
|
||||
let targetUrl;
|
||||
try {
|
||||
targetUrl = new URL(buildOpenCodeUrl(isGlobalStream ? '/global/event' : '/event', ''));
|
||||
} catch {
|
||||
sendMessageStreamWsFrame(socket, { type: 'error', message: 'OpenCode service unavailable' });
|
||||
socket.close(1011, 'OpenCode service unavailable');
|
||||
return;
|
||||
}
|
||||
|
||||
buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n');
|
||||
|
||||
let separatorIndex = buffer.indexOf('\n\n');
|
||||
while (separatorIndex !== -1) {
|
||||
const block = buffer.slice(0, separatorIndex);
|
||||
buffer = buffer.slice(separatorIndex + 2);
|
||||
forwardBlock(block);
|
||||
separatorIndex = buffer.indexOf('\n\n');
|
||||
if (!isGlobalStream && requestedDirectory) {
|
||||
targetUrl.searchParams.set('directory', requestedDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.trim().length > 0) {
|
||||
forwardBlock(buffer.trim());
|
||||
const headers = {
|
||||
Accept: 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
...getOpenCodeAuthHeaders(),
|
||||
};
|
||||
if (lastEventId) {
|
||||
headers['Last-Event-ID'] = lastEventId;
|
||||
}
|
||||
|
||||
const upstreamController = new AbortController();
|
||||
currentUpstreamAbortController = upstreamController;
|
||||
const abortUpstream = () => upstreamController.abort();
|
||||
controller.signal.addEventListener('abort', abortUpstream, { once: true });
|
||||
|
||||
let upstream;
|
||||
try {
|
||||
upstream = await fetchImpl(targetUrl.toString(), {
|
||||
headers,
|
||||
signal: upstreamController.signal,
|
||||
});
|
||||
} catch {
|
||||
controller.signal.removeEventListener('abort', abortUpstream);
|
||||
currentUpstreamAbortController = null;
|
||||
upstreamConnected = false;
|
||||
if (controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
if (!streamReady) {
|
||||
sendMessageStreamWsFrame(socket, { type: 'error', message: 'Failed to connect to OpenCode event stream' });
|
||||
socket.close(1011, 'Failed to connect to OpenCode event stream');
|
||||
triggerHealthCheck?.();
|
||||
return;
|
||||
}
|
||||
await wait(upstreamReconnectDelayMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!upstream.ok || !upstream.body) {
|
||||
controller.signal.removeEventListener('abort', abortUpstream);
|
||||
currentUpstreamAbortController = null;
|
||||
upstreamConnected = false;
|
||||
if (!streamReady) {
|
||||
sendMessageStreamWsFrame(socket, {
|
||||
type: 'error',
|
||||
message: `OpenCode event stream unavailable (${upstream.status})`,
|
||||
});
|
||||
socket.close(1011, 'OpenCode event stream unavailable');
|
||||
if (shouldTriggerUpstreamHealthCheck(upstream)) {
|
||||
triggerHealthCheck?.();
|
||||
}
|
||||
return;
|
||||
}
|
||||
await wait(upstreamReconnectDelayMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!streamReady) {
|
||||
sendMessageStreamWsFrame(socket, {
|
||||
type: 'ready',
|
||||
scope: isGlobalStream ? 'global' : 'directory',
|
||||
});
|
||||
streamReady = true;
|
||||
|
||||
if (isGlobalStream) {
|
||||
wsClients.add(socket);
|
||||
}
|
||||
}
|
||||
|
||||
upstreamConnected = true;
|
||||
const decoder = new TextDecoder();
|
||||
const reader = upstream.body.getReader();
|
||||
let buffer = '';
|
||||
let upstreamAbortReason = null;
|
||||
let stallTimer = null;
|
||||
const resetStallTimer = () => {
|
||||
if (stallTimer) {
|
||||
clearTimeout(stallTimer);
|
||||
}
|
||||
stallTimer = setTimeout(() => {
|
||||
upstreamAbortReason = 'upstream_stalled';
|
||||
upstreamConnected = false;
|
||||
upstreamController.abort();
|
||||
}, upstreamStallTimeoutMs);
|
||||
};
|
||||
|
||||
resetStallTimer();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
resetStallTimer();
|
||||
buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n');
|
||||
|
||||
let separatorIndex = buffer.indexOf('\n\n');
|
||||
while (separatorIndex !== -1) {
|
||||
const block = buffer.slice(0, separatorIndex);
|
||||
buffer = buffer.slice(separatorIndex + 2);
|
||||
forwardBlock(block);
|
||||
separatorIndex = buffer.indexOf('\n\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.trim().length > 0) {
|
||||
forwardBlock(buffer.trim());
|
||||
}
|
||||
} catch (error) {
|
||||
if (!controller.signal.aborted && upstreamAbortReason !== 'upstream_stalled') {
|
||||
console.warn('Message stream WS proxy error:', error);
|
||||
}
|
||||
} finally {
|
||||
if (stallTimer) {
|
||||
clearTimeout(stallTimer);
|
||||
}
|
||||
upstreamConnected = false;
|
||||
currentUpstreamAbortController = null;
|
||||
controller.signal.removeEventListener('abort', abortUpstream);
|
||||
}
|
||||
|
||||
if (controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
|
||||
await wait(upstreamReconnectDelayMs);
|
||||
}
|
||||
} catch (error) {
|
||||
if (!controller.signal.aborted) {
|
||||
|
||||
@@ -1,6 +1,68 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { createGlobalUiEventBroadcaster } from './runtime.js';
|
||||
import { createGlobalUiEventBroadcaster, createMessageStreamWsRuntime } from './runtime.js';
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.readyState = 1;
|
||||
this.sent = [];
|
||||
this.closeCalls = [];
|
||||
}
|
||||
|
||||
send(payload) {
|
||||
this.sent.push(JSON.parse(payload));
|
||||
}
|
||||
|
||||
ping() {
|
||||
void 0;
|
||||
}
|
||||
|
||||
close(code, reason) {
|
||||
if (this.readyState === 3) {
|
||||
return;
|
||||
}
|
||||
this.readyState = 3;
|
||||
this.closeCalls.push({ code, reason });
|
||||
this.emit('close');
|
||||
}
|
||||
}
|
||||
|
||||
function createSseResponse({ blocks = [], signal, holdOpen = false }) {
|
||||
const encoder = new TextEncoder();
|
||||
let index = 0;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
body: {
|
||||
getReader() {
|
||||
return {
|
||||
async read() {
|
||||
if (index < blocks.length) {
|
||||
const next = blocks[index++];
|
||||
return { value: encoder.encode(next), done: false };
|
||||
}
|
||||
|
||||
if (!holdOpen) {
|
||||
return { value: undefined, done: true };
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
const error = new Error('Aborted');
|
||||
error.name = 'AbortError';
|
||||
reject(error);
|
||||
};
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('event stream broadcaster', () => {
|
||||
it('fans out synthetic events to SSE and WS clients', () => {
|
||||
@@ -63,3 +125,71 @@ describe('event stream broadcaster', () => {
|
||||
expect(wsClients.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('message stream websocket runtime', () => {
|
||||
it('reconnects a stalled upstream SSE stream and resumes from the last event id', async () => {
|
||||
const server = new EventEmitter();
|
||||
const wsClients = new Set();
|
||||
let triggerHealthCheckCalls = 0;
|
||||
const fetchCalls = [];
|
||||
let upstreamAttempt = 0;
|
||||
|
||||
const runtime = createMessageStreamWsRuntime({
|
||||
server,
|
||||
uiAuthController: null,
|
||||
isRequestOriginAllowed: async () => true,
|
||||
rejectWebSocketUpgrade() {
|
||||
throw new Error('upgrade should not be used in this test');
|
||||
},
|
||||
buildOpenCodeUrl: (path) => `http://127.0.0.1:4096${path}`,
|
||||
getOpenCodeAuthHeaders: () => ({}),
|
||||
processForwardedEventPayload() {},
|
||||
wsClients,
|
||||
triggerHealthCheck: () => {
|
||||
triggerHealthCheckCalls += 1;
|
||||
},
|
||||
heartbeatIntervalMs: 50,
|
||||
upstreamStallTimeoutMs: 20,
|
||||
upstreamReconnectDelayMs: 0,
|
||||
fetchImpl: async (_url, options) => {
|
||||
const lastEventId = options?.headers?.['Last-Event-ID'] ?? null;
|
||||
fetchCalls.push(lastEventId);
|
||||
upstreamAttempt += 1;
|
||||
|
||||
if (upstreamAttempt === 1) {
|
||||
return createSseResponse({
|
||||
signal: options.signal,
|
||||
holdOpen: true,
|
||||
blocks: [
|
||||
'id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n',
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
return createSseResponse({
|
||||
signal: options.signal,
|
||||
holdOpen: false,
|
||||
blocks: [
|
||||
'id: evt-2\ndata: {"type":"server.connected","properties":{}}\n\n',
|
||||
],
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const socket = new FakeSocket();
|
||||
runtime.wsServer.emit('connection', socket, { url: '/api/global/event/ws' });
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 35));
|
||||
|
||||
const readyFrames = socket.sent.filter((frame) => frame.type === 'ready');
|
||||
const eventFrames = socket.sent.filter((frame) => frame.type === 'event' && frame.payload?.type === 'server.connected');
|
||||
|
||||
expect(readyFrames).toHaveLength(1);
|
||||
expect(eventFrames.length).toBeGreaterThanOrEqual(2);
|
||||
expect(fetchCalls.slice(0, 2)).toEqual([null, 'evt-1']);
|
||||
expect(triggerHealthCheckCalls).toBe(0);
|
||||
|
||||
socket.close();
|
||||
await runtime.close();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user