fix(server): rebind message-stream upstreams after a managed OpenCode restart
When the managed OpenCode process exits but a server survives on the old port (Windows: killProcessOnPort is a no-op, so the orphaned process tree keeps the port), restartOpenCode() times out waiting for the port and spawns a fresh server on a NEW port. HTTP/proxy traffic follows the new port, but the global message-stream hub's upstream SSE reader stays pinned to the old server's /global/event stream — that connection never closes — so new events never reach the UI and the chat stops updating until the app is restarted (#2638). Lifecycle now fires an optional onOpenCodeRestarted hook after a successful managed restart; index.js wires it to the new messageStreamRuntime.rebindUpstream(), which restarts the shared hub (its reader re-dials buildOpenCodeUrl → the current port) and closes directory-scoped sockets so their per-connection readers rebuild against the new port. External servers are untouched (their port cannot change). Fixes #2638
This commit is contained in:
@@ -1079,6 +1079,19 @@ const openCodeLifecycleRuntime = createOpenCodeLifecycleRuntime({
|
||||
}
|
||||
return [...new Set(directories)];
|
||||
},
|
||||
// A managed restart can move OpenCode to a NEW port (the old one may stay
|
||||
// occupied by an orphaned process, e.g. killProcessOnPort is a no-op on
|
||||
// Windows). Rebind the message-stream upstream readers to the current port
|
||||
// so the UI keeps receiving events instead of staying pinned to the old
|
||||
// process (#2638). The runtime is created later by the startup pipeline;
|
||||
// by the time any restart runs, it is assigned.
|
||||
onOpenCodeRestarted: () => {
|
||||
try {
|
||||
messageStreamRuntime?.rebindUpstream();
|
||||
} catch (error) {
|
||||
console.warn('Failed to rebind message stream after OpenCode restart:', error?.message ?? error);
|
||||
}
|
||||
},
|
||||
getManagedOpenCodeEnv: async () => {
|
||||
const settings = await readSettingsFromDiskMigrated().catch(() => null);
|
||||
const managedEnv = settings?.agentControlToolEnabled === false
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { createGlobalMessageStreamHub } from './global-hub.js';
|
||||
import { createMessageStreamWsRuntime } from './runtime.js';
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.readyState = 1;
|
||||
this.sent = [];
|
||||
this.closeCalls = [];
|
||||
}
|
||||
|
||||
send(payload) {
|
||||
this.sent.push(JSON.parse(payload));
|
||||
}
|
||||
|
||||
ping() {
|
||||
void 0;
|
||||
}
|
||||
|
||||
close(code, reason) {
|
||||
if (this.readyState === 3) {
|
||||
return;
|
||||
}
|
||||
this.readyState = 3;
|
||||
this.closeCalls.push({ code, reason });
|
||||
this.emit('close');
|
||||
}
|
||||
}
|
||||
|
||||
function createSseResponse({ blocks = [], signal, holdOpen = false }) {
|
||||
const encoder = new TextEncoder();
|
||||
let index = 0;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
body: {
|
||||
getReader() {
|
||||
return {
|
||||
async read() {
|
||||
if (index < blocks.length) {
|
||||
const next = blocks[index++];
|
||||
return { value: encoder.encode(next), done: false };
|
||||
}
|
||||
|
||||
if (!holdOpen) {
|
||||
return { value: undefined, done: true };
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
const error = new Error('Aborted');
|
||||
error.name = 'AbortError';
|
||||
reject(error);
|
||||
};
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('rebindUpstream (#2638)', () => {
|
||||
it('restarts the shared hub upstream so a connected client resumes receiving events on the new port', async () => {
|
||||
const server = new EventEmitter();
|
||||
const wsClients = new Set();
|
||||
let port = 4096;
|
||||
let fetchCalls = 0;
|
||||
|
||||
// Port changes after a managed restart: buildOpenCodeUrl resolves the
|
||||
// CURRENT port on every attempt, exactly like production network-runtime.
|
||||
const buildOpenCodeUrl = vi.fn(() => `http://127.0.0.1:${port}/global/event`);
|
||||
const fetchImpl = vi.fn(async (_url, options) => {
|
||||
fetchCalls += 1;
|
||||
if (fetchCalls === 1) {
|
||||
return createSseResponse({
|
||||
signal: options.signal,
|
||||
holdOpen: true,
|
||||
blocks: ['id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n'],
|
||||
});
|
||||
}
|
||||
return createSseResponse({
|
||||
signal: options.signal,
|
||||
holdOpen: true,
|
||||
blocks: ['id: evt-2\ndata: {"type":"session.updated","properties":{"sessionID":"ses_1"}}\n\n'],
|
||||
});
|
||||
});
|
||||
|
||||
const globalHub = createGlobalMessageStreamHub({
|
||||
buildOpenCodeUrl,
|
||||
getOpenCodeAuthHeaders: () => ({}),
|
||||
fetchImpl,
|
||||
upstreamReconnectDelayMs: 0,
|
||||
});
|
||||
|
||||
const runtime = createMessageStreamWsRuntime({
|
||||
server,
|
||||
uiAuthController: null,
|
||||
isRequestOriginAllowed: async () => true,
|
||||
rejectWebSocketUpgrade() {
|
||||
throw new Error('upgrade should not be used in this test');
|
||||
},
|
||||
globalEventHub: globalHub,
|
||||
buildOpenCodeUrl,
|
||||
getOpenCodeAuthHeaders: () => ({}),
|
||||
processForwardedEventPayload() {},
|
||||
wsClients,
|
||||
heartbeatIntervalMs: 5000,
|
||||
upstreamReconnectDelayMs: 0,
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
const socket = new FakeSocket();
|
||||
runtime.wsServer.emit('connection', socket, { url: '/api/global/event/ws' });
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(fetchCalls).toBe(1);
|
||||
expect(socket.sent.some((frame) => frame.type === 'event' && frame.eventId === 'evt-1')).toBe(true);
|
||||
|
||||
// The managed process was restarted onto a new port while the old
|
||||
// process's SSE stream stays open (orphaned survivor).
|
||||
port = 5000;
|
||||
runtime.rebindUpstream();
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
// The hub dialed the new port and the connected client received events
|
||||
// from the new upstream without reconnecting its own socket.
|
||||
expect(fetchCalls).toBe(2);
|
||||
expect(fetchImpl.mock.calls[1][0]).toContain(':5000/global/event');
|
||||
expect(socket.sent.some((frame) => frame.type === 'event' && frame.eventId === 'evt-2')).toBe(true);
|
||||
|
||||
socket.close();
|
||||
await runtime.close();
|
||||
});
|
||||
|
||||
it('closes directory-scoped sockets so their pinned readers reconnect to the new port', async () => {
|
||||
const server = new EventEmitter();
|
||||
const wsClients = new Set();
|
||||
let port = 4096;
|
||||
let fetchCalls = 0;
|
||||
|
||||
const buildOpenCodeUrl = vi.fn(() => `http://127.0.0.1:${port}/event`);
|
||||
const fetchImpl = vi.fn(async (_url, options) => {
|
||||
fetchCalls += 1;
|
||||
return createSseResponse({
|
||||
signal: options.signal,
|
||||
holdOpen: true,
|
||||
blocks: ['id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n'],
|
||||
});
|
||||
});
|
||||
|
||||
const runtime = createMessageStreamWsRuntime({
|
||||
server,
|
||||
uiAuthController: null,
|
||||
isRequestOriginAllowed: async () => true,
|
||||
rejectWebSocketUpgrade() {
|
||||
throw new Error('upgrade should not be used in this test');
|
||||
},
|
||||
buildOpenCodeUrl,
|
||||
getOpenCodeAuthHeaders: () => ({}),
|
||||
processForwardedEventPayload() {},
|
||||
wsClients,
|
||||
heartbeatIntervalMs: 5000,
|
||||
upstreamReconnectDelayMs: 0,
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
const directorySocket = new FakeSocket();
|
||||
runtime.wsServer.emit('connection', directorySocket, { url: '/api/event/ws?directory=%2Fproj' });
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(fetchCalls).toBe(1);
|
||||
|
||||
port = 5000;
|
||||
runtime.rebindUpstream();
|
||||
|
||||
expect(directorySocket.readyState).toBe(3);
|
||||
expect(directorySocket.closeCalls.length).toBeGreaterThan(0);
|
||||
|
||||
directorySocket.close();
|
||||
await runtime.close();
|
||||
});
|
||||
});
|
||||
@@ -70,6 +70,12 @@ export function createMessageStreamWsRuntime({
|
||||
noServer: true,
|
||||
});
|
||||
|
||||
// Directory-scoped streams create one upstream reader per client
|
||||
// connection. Track those sockets so a managed OpenCode restart can close
|
||||
// them: each reader is pinned to the port it connected at and would
|
||||
// otherwise keep streaming from an orphaned process on the old port (#2638).
|
||||
const directorySockets = new Set();
|
||||
|
||||
const ownsGlobalHub = !globalEventHub;
|
||||
const globalHub = globalEventHub ?? createGlobalMessageStreamHub({
|
||||
buildOpenCodeUrl,
|
||||
@@ -103,6 +109,11 @@ export function createMessageStreamWsRuntime({
|
||||
return;
|
||||
}
|
||||
|
||||
directorySockets.add(socket);
|
||||
socket.on('close', () => {
|
||||
directorySockets.delete(socket);
|
||||
});
|
||||
|
||||
acceptDirectoryMessageStreamWsConnection({
|
||||
socket,
|
||||
requestedLastEventId,
|
||||
@@ -156,6 +167,27 @@ export function createMessageStreamWsRuntime({
|
||||
|
||||
return {
|
||||
wsServer,
|
||||
/**
|
||||
* Rebind all upstream readers to the current OpenCode port. Called after
|
||||
* a managed process restart: the restart can land on a NEW port while
|
||||
* the old process (or an orphaned survivor of it) still holds the
|
||||
* previous one, and a healthy-but-pinned SSE connection never notices —
|
||||
* so the UI would stop receiving events until the app restarts (#2638).
|
||||
* Restarting the shared hub re-dials `buildOpenCodeUrl` (which reads the
|
||||
* current port) on its next attempt; directory-scoped readers are
|
||||
* rebuilt by closing their client sockets, which reconnect with
|
||||
* `Last-Event-ID` and re-establish the stream against the new port.
|
||||
*/
|
||||
rebindUpstream() {
|
||||
globalHub.stop();
|
||||
globalHub.start();
|
||||
for (const socket of Array.from(directorySockets)) {
|
||||
try {
|
||||
socket.close(1012, 'OpenCode upstream restarted');
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
},
|
||||
async close() {
|
||||
server.off('upgrade', upgradeHandler);
|
||||
globalBridge.close();
|
||||
|
||||
@@ -48,6 +48,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
getActiveSessionCount = () => 0,
|
||||
reapManagedOrphanedProcesses = reapOrphanedProcesses,
|
||||
getWarmupDirectories = async () => [],
|
||||
onOpenCodeRestarted = null,
|
||||
now = Date.now,
|
||||
} = deps;
|
||||
|
||||
@@ -694,6 +695,17 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
setupProxy(state.expressApp);
|
||||
ensureOpenCodeApiPrefix();
|
||||
}
|
||||
|
||||
// The restart may have landed on a NEW port (the old one can remain
|
||||
// occupied by an orphaned process, e.g. Windows killProcessOnPort is a
|
||||
// no-op). Upstream event readers pinned to the old process would keep
|
||||
// the UI silent forever, so rebind them to the current port. Best
|
||||
// effort: a failure here must not fail the restart itself.
|
||||
try {
|
||||
onOpenCodeRestarted?.();
|
||||
} catch (error) {
|
||||
console.warn('Failed to rebind event stream after OpenCode restart:', error?.message ?? error);
|
||||
}
|
||||
})();
|
||||
|
||||
try {
|
||||
|
||||
@@ -286,6 +286,70 @@ describe('OpenCode lifecycle', () => {
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('calls onOpenCodeRestarted after a successful managed restart', async () => {
|
||||
const close = vi.fn(async () => {});
|
||||
const replacement = createMockChild();
|
||||
const onOpenCodeRestarted = vi.fn();
|
||||
globalThis.fetch = vi.fn(async () => ({
|
||||
ok: false,
|
||||
json: async () => null,
|
||||
}));
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
replacement.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n');
|
||||
});
|
||||
return replacement;
|
||||
});
|
||||
const runtime = createRuntime({ onOpenCodeRestarted }, {
|
||||
openCodePort: 45678,
|
||||
openCodeProcess: {
|
||||
pid: null,
|
||||
exitCode: 1,
|
||||
signalCode: null,
|
||||
close,
|
||||
},
|
||||
});
|
||||
|
||||
await runtime.triggerHealthCheck();
|
||||
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||
// The restart completed on a (possibly new) port — the event-stream
|
||||
// upstreams must rebind so the UI keeps receiving events (#2638).
|
||||
expect(onOpenCodeRestarted).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not call onOpenCodeRestarted when a managed restart fails', async () => {
|
||||
const close = vi.fn(async () => {});
|
||||
const onOpenCodeRestarted = vi.fn();
|
||||
globalThis.fetch = vi.fn(async () => ({
|
||||
ok: false,
|
||||
json: async () => null,
|
||||
}));
|
||||
spawnMock.mockImplementation(() => {
|
||||
const child = createMockChild();
|
||||
queueMicrotask(() => {
|
||||
child.emit('error', new Error('spawn failed'));
|
||||
});
|
||||
return child;
|
||||
});
|
||||
const runtime = createRuntime({ onOpenCodeRestarted }, {
|
||||
openCodePort: 45678,
|
||||
openCodeProcess: {
|
||||
pid: null,
|
||||
exitCode: 1,
|
||||
signalCode: null,
|
||||
close,
|
||||
},
|
||||
});
|
||||
|
||||
// triggerHealthCheck logs instead of rethrowing; call restartOpenCode
|
||||
// directly to observe the failure result.
|
||||
await expect(runtime.restartOpenCode()).rejects.toThrow();
|
||||
|
||||
expect(onOpenCodeRestarted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('launches managed OpenCode with the managed PATH', async () => {
|
||||
delete process.env.OPENCODE_BINARY;
|
||||
const child = createMockChild();
|
||||
|
||||
Reference in New Issue
Block a user