fix: settle busy sessions after managed OpenCode restart (#3002)
* fix: reconcile busy sessions after managed OpenCode restart Forced health-check restarts previously rebound the event stream without settling in-flight turns, so sessions stayed busy with no terminal state. Interrupt those sessions, classify health failures, and retain bounded process diagnostics for post-restart diagnosis. Fixes #2943 Co-authored-by: serkraser <serkraser@gmail.com> * fix: surface interrupted chats after OpenCode restart Complete unfinished assistant turns as aborted once the session is authoritatively idle, and show a persistent toast so users can continue instead of remaining silently stranded. Fixes #2943 Co-authored-by: serkraser <serkraser@gmail.com> * fix: redact Basic auth credentials in restart diagnostics The key/value sanitizer stopped at whitespace, so Authorization: Basic credentials survived in stderr tails and health snapshots. Redact the scheme token before that rule runs. Co-authored-by: serkraser <serkraser@gmail.com>
This commit is contained in:
@@ -111,12 +111,13 @@ This module provides OpenCode server integration utilities for the web server ru
|
||||
- `markSessionUnviewed(sessionId, clientId)`
|
||||
- `markUserMessageSent(sessionId)`
|
||||
- `resetAllSessionActivityToIdle()`
|
||||
- `interruptBusySessionsAfterRestart()`: settles every session whose authoritative status is `busy`/`retry` or whose activity phase is still busy, broadcasts `openchamber:session-status` idle plus an OpenCode-shaped `session.error`, resets leftover activity/cooldowns, and returns the interrupted session IDs in stable order.
|
||||
- `dispose()`
|
||||
|
||||
The runtime maintains active-session count incrementally from idempotent activity phase transitions. Upstream stall-timeout and lifecycle health checks read it in O(1); the hourly cleanup removes activity phases older than 24 hours without broadcasting synthetic state transitions. Snapshot generation remains reserved for the session-activity API.
|
||||
|
||||
## Public exports (lifecycle.js)
|
||||
- `createOpenCodeLifecycleRuntime(dependencies)`: creates lifecycle runtime for managed/external OpenCode process orchestration. The optional `onOpenCodeRestarted` dependency (default `null`) is fired after a successful managed restart; `index.js` wires it to `messageStreamRuntime.rebindUpstream()` so event-stream readers rebind to the possibly-new port (a restart can land on a new port while an orphaned process keeps the old one, which would otherwise leave the chat UI silent — issue #2638).
|
||||
- `createOpenCodeLifecycleRuntime(dependencies)`: creates lifecycle runtime for managed/external OpenCode process orchestration. The optional `onOpenCodeRestarted` dependency (default `null`) is fired after a successful managed restart. `index.js` rebinds event-stream readers to the possibly-new port (#2638), then calls `interruptBusySessionsAfterRestart()` and broadcasts one `opencode-restart-interrupted` UI notification when interrupted turns exist (#2943).
|
||||
- Returned API:
|
||||
- `startOpenCode()`
|
||||
- `restartOpenCode()`
|
||||
@@ -147,6 +148,8 @@ macOS `say` voice enumeration starts concurrently with server composition. The s
|
||||
|
||||
Transport-triggered health checks share the periodic monitor's failure accounting interval. Rapid WS reconnect callbacks therefore cannot exhaust the managed-process restart threshold using one cached unhealthy result; an exited managed process still restarts immediately.
|
||||
|
||||
Managed health failures are classified as `timeout`, `connection_refused`, `connection_reset`, `invalid_response`, or `error`. The lifecycle retains the latest counted failure with a bounded detail string and source. Managed process wrappers continue capturing a sanitized, bounded stderr tail after readiness and retain exit code/signal. Before replacing a managed process, lifecycle snapshots the reason, latest health failure, process diagnostics/aliveness, busy-session count, and timestamp into `lastOpenCodeRestartDiagnostics`; successful startup does not clear this snapshot, and `/health` exposes it for post-restart diagnosis without process environment or credentials.
|
||||
|
||||
## Public exports (env-runtime.js)
|
||||
- `createOpenCodeEnvRuntime(dependencies)`: creates runtime that owns OpenCode CLI environment and binary discovery state.
|
||||
- OpenCode CLI resolution order is persisted settings, environment overrides, bundled Desktop CLI when available, PATH, known install locations, then platform shell discovery.
|
||||
|
||||
@@ -22,6 +22,65 @@ const OPENCODE_HEALTH_PATH = '/global/health';
|
||||
// tails are unlikely to be the user's first click and just add background work.
|
||||
const WARMUP_DIRECTORY_LIMIT = 4;
|
||||
const WARMUP_REQUEST_TIMEOUT_MS = 30000;
|
||||
const MANAGED_STDERR_TAIL_MAX_BYTES = 32 * 1024;
|
||||
const HEALTH_FAILURE_DETAIL_MAX_LENGTH = 256;
|
||||
|
||||
const getBoundedTextTail = (value, maxBytes) => {
|
||||
const buffer = Buffer.from(String(value ?? ''));
|
||||
if (buffer.byteLength <= maxBytes) return buffer.toString();
|
||||
return buffer.subarray(buffer.byteLength - maxBytes).toString();
|
||||
};
|
||||
|
||||
const sanitizeDiagnosticText = (value) => String(value ?? '')
|
||||
.replace(/(https?:\/\/)[^/\s:@]+:[^/\s@]+@/gi, '$1[redacted]@')
|
||||
.replace(/\b(Bearer)\s+[^\s,;]+/gi, '$1 [redacted]')
|
||||
// Unquoted `Authorization: <scheme> <credential>` values must be handled
|
||||
// before the generic key/value rule below: that rule stops at whitespace, so
|
||||
// it would redact only the scheme word and leave the credential intact.
|
||||
// Scoped to authorization-style keys so ordinary prose using "basic" or
|
||||
// "token" is not mangled.
|
||||
.replace(
|
||||
/(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}authorization[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*(?:"|')?(?:basic|bearer|token)\s+)[^\s,;"']+/gim,
|
||||
'$1$2[redacted]',
|
||||
)
|
||||
.replace(/([?&][^=&#\s]*(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[^=&#\s]*=)[^&#\s]+/gi, '$1[redacted]')
|
||||
.replace(
|
||||
/(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;]+)/gim,
|
||||
'$1$2[redacted]',
|
||||
);
|
||||
|
||||
const getHealthFailureDetail = (error) => {
|
||||
const name = String(error?.name || 'Error');
|
||||
const message = String(error?.message || error || 'Unknown error');
|
||||
return sanitizeDiagnosticText(`${name}: ${message}`).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH);
|
||||
};
|
||||
|
||||
const classifyHealthProbeError = (error) => {
|
||||
const name = String(error?.name || '');
|
||||
const code = String(error?.code || '').toUpperCase();
|
||||
const message = String(error?.message || error || '');
|
||||
const normalizedMessage = message.toLowerCase();
|
||||
|
||||
if (
|
||||
name === 'AbortError'
|
||||
|| name === 'TimeoutError'
|
||||
|| normalizedMessage.includes('the operation was aborted')
|
||||
|| normalizedMessage.includes('abortsignal.timeout')
|
||||
) {
|
||||
return { class: 'timeout', detail: getHealthFailureDetail(error) };
|
||||
}
|
||||
if (code === 'ECONNREFUSED' || normalizedMessage.includes('econnrefused')) {
|
||||
return { class: 'connection_refused', detail: getHealthFailureDetail(error) };
|
||||
}
|
||||
if (
|
||||
code === 'ECONNRESET'
|
||||
|| normalizedMessage.includes('econnreset')
|
||||
|| normalizedMessage.includes('socket hang up')
|
||||
) {
|
||||
return { class: 'connection_reset', detail: getHealthFailureDetail(error) };
|
||||
}
|
||||
return { class: 'error', detail: getHealthFailureDetail(error) };
|
||||
};
|
||||
|
||||
export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
const {
|
||||
@@ -88,6 +147,36 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
}
|
||||
};
|
||||
|
||||
const snapshotManagedOpenCodeProcess = (child = state.openCodeProcess) => {
|
||||
if (!child) return null;
|
||||
const snapshot = {
|
||||
pid: child.pid || null,
|
||||
exitCode: child.exitCode ?? null,
|
||||
signalCode: child.signalCode ?? null,
|
||||
stderrTail: getBoundedTextTail(
|
||||
sanitizeDiagnosticText(child.stderrTail ?? ''),
|
||||
MANAGED_STDERR_TAIL_MAX_BYTES,
|
||||
),
|
||||
};
|
||||
state.lastManagedOpenCodeProcess = snapshot;
|
||||
return snapshot;
|
||||
};
|
||||
|
||||
const captureRestartDiagnostics = (reason) => {
|
||||
const processSnapshot = snapshotManagedOpenCodeProcess();
|
||||
const diagnostics = {
|
||||
reason: sanitizeDiagnosticText(String(reason || 'managed-restart')).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH),
|
||||
healthFailure: state.lastOpenCodeHealthFailure ? { ...state.lastOpenCodeHealthFailure } : null,
|
||||
process: processSnapshot
|
||||
? { ...processSnapshot, alive: isManagedOpenCodeProcessAlive() }
|
||||
: null,
|
||||
busySessionCount: getActiveSessionCount(),
|
||||
at: new Date(now()).toISOString(),
|
||||
};
|
||||
state.lastOpenCodeRestartDiagnostics = diagnostics;
|
||||
console.warn('[lifecycle] managed OpenCode restart diagnostics', diagnostics);
|
||||
};
|
||||
|
||||
const waitForChildProcessClose = (child, timeoutMs) => new Promise((resolve) => {
|
||||
if (!child || hasChildProcessExited(child)) {
|
||||
resolve(true);
|
||||
@@ -297,6 +386,34 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
windowsHide: true,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let runtimeStderrTail = '';
|
||||
let runtimeStderrAttached = false;
|
||||
let observedExitCode = null;
|
||||
let observedSignalCode = null;
|
||||
|
||||
const getManagedProcessSnapshot = () => ({
|
||||
pid: child.pid || null,
|
||||
exitCode: observedExitCode ?? child.exitCode ?? null,
|
||||
signalCode: observedSignalCode ?? child.signalCode ?? null,
|
||||
stderrTail: getBoundedTextTail(sanitizeDiagnosticText(runtimeStderrTail), MANAGED_STDERR_TAIL_MAX_BYTES),
|
||||
});
|
||||
const recordManagedProcessExit = (code, signal) => {
|
||||
if (code !== null && code !== undefined) observedExitCode = code;
|
||||
if (signal !== null && signal !== undefined) observedSignalCode = signal;
|
||||
state.lastManagedOpenCodeProcess = getManagedProcessSnapshot();
|
||||
};
|
||||
const attachRuntimeStderrCapture = () => {
|
||||
if (runtimeStderrAttached) return;
|
||||
runtimeStderrAttached = true;
|
||||
child.stderr?.on('data', (chunk) => {
|
||||
runtimeStderrTail = getBoundedTextTail(
|
||||
`${runtimeStderrTail}${chunk.toString()}`,
|
||||
MANAGED_STDERR_TAIL_MAX_BYTES,
|
||||
);
|
||||
});
|
||||
};
|
||||
child.on('exit', recordManagedProcessExit);
|
||||
child.on('close', recordManagedProcessExit);
|
||||
|
||||
const url = await new Promise((resolve, reject) => {
|
||||
let stdout = '';
|
||||
@@ -323,6 +440,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
finish(reject, new Error(`Failed to parse server url from output: ${line}`));
|
||||
return;
|
||||
}
|
||||
attachRuntimeStderrCapture();
|
||||
finish(resolve, match[1]);
|
||||
return;
|
||||
}
|
||||
@@ -371,10 +489,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
url,
|
||||
pid: child.pid || null,
|
||||
get exitCode() {
|
||||
return child.exitCode;
|
||||
return observedExitCode ?? child.exitCode;
|
||||
},
|
||||
get signalCode() {
|
||||
return child.signalCode;
|
||||
return observedSignalCode ?? child.signalCode;
|
||||
},
|
||||
get stderrTail() {
|
||||
return getManagedProcessSnapshot().stderrTail;
|
||||
},
|
||||
async close() {
|
||||
await closeManagedOpenCodeChild(child);
|
||||
@@ -416,9 +537,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
});
|
||||
};
|
||||
|
||||
const isOpenCodeProcessHealthy = async () => {
|
||||
const probeOpenCodeHealthDetailed = async () => {
|
||||
if (!state.openCodeProcess || !state.openCodePort) {
|
||||
return false;
|
||||
return {
|
||||
healthy: false,
|
||||
failure: {
|
||||
class: 'error',
|
||||
detail: 'Managed OpenCode process or port is unavailable',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -430,14 +557,47 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
},
|
||||
signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) return false;
|
||||
const body = await response.json().catch(() => null);
|
||||
return body?.healthy === true;
|
||||
} catch {
|
||||
return false;
|
||||
if (!response.ok) {
|
||||
return {
|
||||
healthy: false,
|
||||
failure: {
|
||||
class: 'invalid_response',
|
||||
detail: `Health endpoint returned HTTP ${response.status ?? 'unknown'}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
let body;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
return {
|
||||
healthy: false,
|
||||
failure: {
|
||||
class: 'invalid_response',
|
||||
detail: 'Health endpoint returned invalid JSON',
|
||||
},
|
||||
};
|
||||
}
|
||||
if (body?.healthy !== true) {
|
||||
return {
|
||||
healthy: false,
|
||||
failure: {
|
||||
class: 'invalid_response',
|
||||
detail: 'Health endpoint did not report healthy=true',
|
||||
},
|
||||
};
|
||||
}
|
||||
return { healthy: true, failure: null };
|
||||
} catch (error) {
|
||||
return {
|
||||
healthy: false,
|
||||
failure: classifyHealthProbeError(error),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const isOpenCodeProcessHealthy = async () => (await probeOpenCodeHealthDetailed()).healthy;
|
||||
|
||||
const probeExternalOpenCode = async (port, origin) => {
|
||||
if (!port || port <= 0) {
|
||||
return false;
|
||||
@@ -617,7 +777,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
throw lastError;
|
||||
};
|
||||
|
||||
const restartOpenCode = async () => {
|
||||
const restartOpenCode = async (reason = 'managed-restart') => {
|
||||
if (state.isShuttingDown) return;
|
||||
if (state.currentRestartPromise) {
|
||||
await state.currentRestartPromise;
|
||||
@@ -655,6 +815,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
return;
|
||||
}
|
||||
|
||||
captureRestartDiagnostics(reason);
|
||||
const portToKill = state.openCodePort;
|
||||
|
||||
if (state.openCodeProcess) {
|
||||
@@ -820,7 +981,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
clearResolvedOpenCodeBinary();
|
||||
await applyOpencodeBinaryFromSettings();
|
||||
|
||||
await restartOpenCode();
|
||||
await restartOpenCode(reason || 'config-change');
|
||||
|
||||
// A managed OpenCode process is restarted (and thus re-reads config from
|
||||
// disk) by restartOpenCode(). An external OpenCode server is NOT owned by
|
||||
@@ -1010,17 +1171,17 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
const probeOpenCodeHealth = async () => {
|
||||
const checkedAt = now();
|
||||
if (lastHealthProbeResult && checkedAt - lastHealthProbeResult.at < HEALTH_CHECK_RESULT_CACHE_MS) {
|
||||
return lastHealthProbeResult.healthy;
|
||||
return lastHealthProbeResult;
|
||||
}
|
||||
|
||||
if (healthProbePromise) {
|
||||
return healthProbePromise;
|
||||
}
|
||||
|
||||
healthProbePromise = isOpenCodeProcessHealthy()
|
||||
.then((healthy) => {
|
||||
lastHealthProbeResult = { at: now(), healthy };
|
||||
return healthy;
|
||||
healthProbePromise = probeOpenCodeHealthDetailed()
|
||||
.then((result) => {
|
||||
lastHealthProbeResult = { at: now(), ...result };
|
||||
return lastHealthProbeResult;
|
||||
})
|
||||
.finally(() => {
|
||||
healthProbePromise = null;
|
||||
@@ -1033,13 +1194,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
const activeCount = getActiveSessionCount();
|
||||
if (activeCount === 0) {
|
||||
lastUnhealthyWithBusySessionsAt = 0;
|
||||
return false;
|
||||
return { skip: false, staleBusy: false };
|
||||
}
|
||||
|
||||
const checkedAt = now();
|
||||
if (!lastUnhealthyWithBusySessionsAt) {
|
||||
lastUnhealthyWithBusySessionsAt = checkedAt;
|
||||
return true;
|
||||
return { skip: true, staleBusy: false };
|
||||
}
|
||||
|
||||
if (checkedAt - lastUnhealthyWithBusySessionsAt >= STALE_BUSY_GRACE_MS) {
|
||||
@@ -1047,10 +1208,10 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
`[lifecycle] OpenCode unhealthy with ${activeCount} busy session(s) for > 2 min — forcing restart`
|
||||
);
|
||||
lastUnhealthyWithBusySessionsAt = 0;
|
||||
return false;
|
||||
return { skip: false, staleBusy: true };
|
||||
}
|
||||
|
||||
return true;
|
||||
return { skip: true, staleBusy: false };
|
||||
};
|
||||
|
||||
const runHealthCheckCycle = async (source) => {
|
||||
@@ -1058,13 +1219,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
if (healthCheckCyclePromise) return healthCheckCyclePromise;
|
||||
|
||||
healthCheckCyclePromise = (async () => {
|
||||
const healthy = await probeOpenCodeHealth();
|
||||
if (!healthy) {
|
||||
const healthResult = await probeOpenCodeHealth();
|
||||
if (!healthResult.healthy) {
|
||||
if (!isManagedOpenCodeProcessAlive()) {
|
||||
console.log(`[lifecycle] ${source} health check: OpenCode process exited, restarting...`);
|
||||
consecutiveHealthFailures = 0;
|
||||
lastHealthProbeResult = null;
|
||||
await restartOpenCode();
|
||||
await restartOpenCode(`${source}-process-exited`);
|
||||
return;
|
||||
}
|
||||
const checkedAt = now();
|
||||
@@ -1073,15 +1234,30 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
|
||||
}
|
||||
lastCountedHealthFailureAt = checkedAt;
|
||||
consecutiveHealthFailures += 1;
|
||||
const healthFailure = healthResult.failure || {
|
||||
class: 'error',
|
||||
detail: 'Health check failed without diagnostic detail',
|
||||
};
|
||||
state.lastOpenCodeHealthFailure = {
|
||||
class: healthFailure.class,
|
||||
detail: healthFailure.detail,
|
||||
at: new Date(checkedAt).toISOString(),
|
||||
source,
|
||||
};
|
||||
console.warn(
|
||||
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES})`
|
||||
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES}) class=${healthFailure.class}`
|
||||
);
|
||||
if (consecutiveHealthFailures < HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES) return;
|
||||
if (shouldSkipRestartForBusySessions()) return;
|
||||
const busyDecision = shouldSkipRestartForBusySessions();
|
||||
if (busyDecision.skip) return;
|
||||
console.log(`[lifecycle] ${source} health check failure threshold reached, restarting OpenCode...`);
|
||||
consecutiveHealthFailures = 0;
|
||||
lastHealthProbeResult = null;
|
||||
await restartOpenCode();
|
||||
await restartOpenCode(
|
||||
busyDecision.staleBusy
|
||||
? `${source}-stale-busy-health-failure`
|
||||
: `${source}-health-failure`,
|
||||
);
|
||||
} else {
|
||||
resetHealthFailureState();
|
||||
}
|
||||
|
||||
@@ -62,6 +62,9 @@ const createRuntime = (overrides = {}, stateOverrides = {}, envOverrides = {}) =
|
||||
openCodeApiPrefixDetected: false,
|
||||
openCodeApiDetectionTimer: null,
|
||||
lastOpenCodeError: null,
|
||||
lastOpenCodeHealthFailure: null,
|
||||
lastManagedOpenCodeProcess: null,
|
||||
lastOpenCodeRestartDiagnostics: null,
|
||||
isOpenCodeReady: false,
|
||||
openCodeNotReadySince: 0,
|
||||
isExternalOpenCode: false,
|
||||
@@ -75,7 +78,7 @@ const createRuntime = (overrides = {}, stateOverrides = {}, envOverrides = {}) =
|
||||
...stateOverrides,
|
||||
};
|
||||
|
||||
return createOpenCodeLifecycleRuntime({
|
||||
const runtime = createOpenCodeLifecycleRuntime({
|
||||
state,
|
||||
env: {
|
||||
ENV_CONFIGURED_OPENCODE_PORT: 45678,
|
||||
@@ -111,6 +114,8 @@ const createRuntime = (overrides = {}, stateOverrides = {}, envOverrides = {}) =
|
||||
})),
|
||||
...overrides,
|
||||
});
|
||||
runtime.testState = state;
|
||||
return runtime;
|
||||
};
|
||||
|
||||
describe('OpenCode lifecycle', () => {
|
||||
@@ -234,6 +239,61 @@ describe('OpenCode lifecycle', () => {
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: 'timeout',
|
||||
expectedClass: 'timeout',
|
||||
fetchResult: () => {
|
||||
const error = new Error('The operation was aborted');
|
||||
error.name = 'AbortError';
|
||||
throw error;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'connection refusal',
|
||||
expectedClass: 'connection_refused',
|
||||
fetchResult: () => {
|
||||
const error = new Error('connect ECONNREFUSED 127.0.0.1:45678');
|
||||
error.code = 'ECONNREFUSED';
|
||||
throw error;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'invalid JSON',
|
||||
expectedClass: 'invalid_response',
|
||||
fetchResult: () => ({
|
||||
ok: true,
|
||||
json: async () => {
|
||||
throw new SyntaxError('Unexpected token');
|
||||
},
|
||||
}),
|
||||
},
|
||||
])('classifies and stores a counted $name health failure', async ({ expectedClass, fetchResult }) => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
globalThis.fetch = vi.fn(fetchResult);
|
||||
const runtime = createRuntime({}, {
|
||||
openCodePort: 45678,
|
||||
openCodeProcess: {
|
||||
pid: process.pid,
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
close: vi.fn(async () => {}),
|
||||
},
|
||||
isOpenCodeReady: true,
|
||||
});
|
||||
|
||||
await runtime.triggerHealthCheck();
|
||||
|
||||
expect(runtime.testState.lastOpenCodeHealthFailure).toEqual({
|
||||
class: expectedClass,
|
||||
detail: expect.any(String),
|
||||
at: expect.any(String),
|
||||
source: 'immediate',
|
||||
});
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining(`class=${expectedClass}`));
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it('does not mistake a live managed process wrapper for an exited child', async () => {
|
||||
const close = vi.fn(async () => {});
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
@@ -320,6 +380,124 @@ describe('OpenCode lifecycle', () => {
|
||||
expect(onOpenCodeRestarted).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('retains post-listen stderr and exited process diagnostics across restart', async () => {
|
||||
const firstChild = createMockChild();
|
||||
const replacement = createMockChild();
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
globalThis.fetch = vi.fn(async () => ({
|
||||
ok: false,
|
||||
status: 503,
|
||||
json: async () => null,
|
||||
}));
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
firstChild.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n');
|
||||
});
|
||||
return firstChild;
|
||||
});
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
replacement.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n');
|
||||
});
|
||||
return replacement;
|
||||
});
|
||||
const runtime = createRuntime();
|
||||
const server = await runtime.startOpenCode();
|
||||
runtime.testState.openCodeProcess = server;
|
||||
|
||||
firstChild.stderr.emit(
|
||||
'data',
|
||||
`${'x'.repeat(40 * 1024)}\ntoken=runtime-secret\nruntime worker failed after startup\n`,
|
||||
);
|
||||
firstChild.exitCode = 7;
|
||||
firstChild.emit('exit', 7, null);
|
||||
|
||||
expect(server.exitCode).toBe(7);
|
||||
expect(Buffer.byteLength(server.stderrTail)).toBeLessThanOrEqual(32 * 1024);
|
||||
expect(server.stderrTail).not.toContain('runtime-secret');
|
||||
expect(server.stderrTail).toContain('runtime worker failed after startup');
|
||||
|
||||
await runtime.triggerHealthCheck();
|
||||
|
||||
expect(runtime.testState.lastOpenCodeRestartDiagnostics).toEqual({
|
||||
reason: 'immediate-process-exited',
|
||||
healthFailure: null,
|
||||
process: {
|
||||
pid: 12345,
|
||||
exitCode: 7,
|
||||
signalCode: null,
|
||||
stderrTail: expect.stringContaining('runtime worker failed after startup'),
|
||||
alive: false,
|
||||
},
|
||||
busySessionCount: 0,
|
||||
at: expect.any(String),
|
||||
});
|
||||
expect(runtime.testState.lastManagedOpenCodeProcess).toEqual({
|
||||
pid: 12345,
|
||||
exitCode: 7,
|
||||
signalCode: null,
|
||||
stderrTail: expect.stringContaining('runtime worker failed after startup'),
|
||||
});
|
||||
|
||||
await runtime.testState.openCodeProcess.close();
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it('redacts Authorization scheme credentials from stderr diagnostics', async () => {
|
||||
const firstChild = createMockChild();
|
||||
const replacement = createMockChild();
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
globalThis.fetch = vi.fn(async () => ({
|
||||
ok: false,
|
||||
status: 503,
|
||||
json: async () => null,
|
||||
}));
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
firstChild.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n');
|
||||
});
|
||||
return firstChild;
|
||||
});
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
replacement.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n');
|
||||
});
|
||||
return replacement;
|
||||
});
|
||||
const runtime = createRuntime();
|
||||
const server = await runtime.startOpenCode();
|
||||
runtime.testState.openCodeProcess = server;
|
||||
|
||||
firstChild.stderr.emit(
|
||||
'data',
|
||||
'request rejected: Authorization: Basic dXNlcjpwYXNz\n'
|
||||
+ 'authorization: basic bG93ZXI6Y2FzZQ==\n'
|
||||
+ 'Authorization: Bearer fake-bearer-token-value\n'
|
||||
+ 'falling back to basic health monitor\n'
|
||||
+ 'runtime worker failed after startup\n',
|
||||
);
|
||||
firstChild.exitCode = 7;
|
||||
firstChild.emit('exit', 7, null);
|
||||
|
||||
expect(server.stderrTail).not.toContain('dXNlcjpwYXNz');
|
||||
expect(server.stderrTail).not.toContain('bG93ZXI6Y2FzZQ');
|
||||
expect(server.stderrTail).not.toContain('fake-bearer-token-value');
|
||||
expect(server.stderrTail).toContain('falling back to basic health monitor');
|
||||
expect(server.stderrTail).toContain('runtime worker failed after startup');
|
||||
|
||||
await runtime.triggerHealthCheck();
|
||||
|
||||
const diagnosticsTail = runtime.testState.lastOpenCodeRestartDiagnostics.process.stderrTail;
|
||||
expect(diagnosticsTail).not.toContain('dXNlcjpwYXNz');
|
||||
expect(diagnosticsTail).not.toContain('bG93ZXI6Y2FzZQ');
|
||||
expect(diagnosticsTail).not.toContain('fake-bearer-token-value');
|
||||
expect(diagnosticsTail).toContain('falling back to basic health monitor');
|
||||
expect(diagnosticsTail).toContain('runtime worker failed after startup');
|
||||
|
||||
await runtime.testState.openCodeProcess.close();
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it('does not call onOpenCodeRestarted when a managed restart fails', async () => {
|
||||
const close = vi.fn(async () => {});
|
||||
const onOpenCodeRestarted = vi.fn();
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { createSessionRuntime } from './session-runtime.js';
|
||||
|
||||
describe('managed OpenCode restart session recovery', () => {
|
||||
it('settles busy sessions and broadcasts one interruption notification', () => {
|
||||
const events = [];
|
||||
const broadcastUiNotification = vi.fn();
|
||||
const rebindUpstream = vi.fn();
|
||||
const sessionRuntime = createSessionRuntime({
|
||||
writeSseEvent() {},
|
||||
getNotificationClients: () => new Set(),
|
||||
broadcastEvent: (event) => events.push(event),
|
||||
});
|
||||
const onOpenCodeRestarted = () => {
|
||||
rebindUpstream();
|
||||
const { sessionIds } = sessionRuntime.interruptBusySessionsAfterRestart();
|
||||
if (sessionIds.length > 0) {
|
||||
const multiple = sessionIds.length > 1;
|
||||
broadcastUiNotification({
|
||||
title: multiple ? 'Chats interrupted' : 'Chat interrupted',
|
||||
body: multiple
|
||||
? 'OpenCode restarted during running responses. Send a message in each chat to continue.'
|
||||
: 'OpenCode restarted during a running response. Send a message to continue.',
|
||||
tag: 'opencode-restart-interrupted',
|
||||
kind: 'opencode-restart-interrupted',
|
||||
sessionId: sessionIds[0],
|
||||
});
|
||||
}
|
||||
};
|
||||
const markBusy = (sessionID) => sessionRuntime.processOpenCodeSsePayload({
|
||||
type: 'session.status',
|
||||
properties: { sessionID, status: { type: 'busy' } },
|
||||
});
|
||||
|
||||
try {
|
||||
markBusy('session-1');
|
||||
markBusy('session-2');
|
||||
markBusy('session-3');
|
||||
events.length = 0;
|
||||
|
||||
onOpenCodeRestarted();
|
||||
|
||||
expect(rebindUpstream).toHaveBeenCalledOnce();
|
||||
expect(sessionRuntime.getActiveSessionCount()).toBe(0);
|
||||
expect(Object.values(sessionRuntime.getSessionStateSnapshot()).map((state) => state.status))
|
||||
.toEqual(['idle', 'idle', 'idle']);
|
||||
expect(events.filter((event) => event.type === 'openchamber:session-status')).toHaveLength(3);
|
||||
expect(events.filter((event) => event.type === 'session.error')).toHaveLength(3);
|
||||
expect(broadcastUiNotification).toHaveBeenCalledOnce();
|
||||
expect(broadcastUiNotification).toHaveBeenCalledWith(expect.objectContaining({
|
||||
kind: 'opencode-restart-interrupted',
|
||||
sessionId: 'session-1',
|
||||
}));
|
||||
} finally {
|
||||
sessionRuntime.dispose();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -130,7 +130,8 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br
|
||||
const now = Date.now();
|
||||
const existing = sessionStates.get(sessionId);
|
||||
const existingAttentionState = sessionAttentionStates.get(sessionId);
|
||||
if (existing && existing.lastUpdateAt > now - 5000 && status === existing.status) {
|
||||
const isRestartInterruption = metadata.reason === 'opencode-restart';
|
||||
if (existing && existing.lastUpdateAt > now - 5000 && status === existing.status && !isRestartInterruption) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -145,7 +146,7 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br
|
||||
const attentionState = sessionAttentionStates.get(sessionId);
|
||||
const attentionChanged = !!attentionState && existingAttentionState?.needsAttention !== attentionState.needsAttention;
|
||||
const clients = getNotificationClients();
|
||||
if (!existing || existing.status !== status || attentionChanged) {
|
||||
if (!existing || existing.status !== status || attentionChanged || isRestartInterruption) {
|
||||
const state = sessionStates.get(sessionId);
|
||||
const syntheticPayload = {
|
||||
type: 'openchamber:session-status',
|
||||
@@ -293,6 +294,41 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br
|
||||
}
|
||||
};
|
||||
|
||||
const interruptBusySessionsAfterRestart = () => {
|
||||
const interruptedSessionIds = new Set();
|
||||
for (const [sessionId, state] of sessionStates) {
|
||||
if (state.status === 'busy' || state.status === 'retry') {
|
||||
interruptedSessionIds.add(sessionId);
|
||||
}
|
||||
}
|
||||
for (const [sessionId, activity] of sessionActivityPhases) {
|
||||
if (activity.phase === 'busy') {
|
||||
interruptedSessionIds.add(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
const eventId = `opencode-restart-${Date.now()}`;
|
||||
for (const sessionId of interruptedSessionIds) {
|
||||
updateSessionState(sessionId, 'idle', eventId, {
|
||||
message: 'Interrupted by OpenCode restart',
|
||||
reason: 'opencode-restart',
|
||||
});
|
||||
broadcastEvent?.({
|
||||
type: 'session.error',
|
||||
properties: {
|
||||
sessionID: sessionId,
|
||||
error: {
|
||||
name: 'MessageAbortedError',
|
||||
message: 'The running turn was interrupted when OpenCode restarted.',
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
resetAllSessionActivityToIdle();
|
||||
return { sessionIds: [...interruptedSessionIds] };
|
||||
};
|
||||
|
||||
const cleanupOldSessionStates = () => {
|
||||
const now = Date.now();
|
||||
for (const [sessionId, data] of sessionStates) {
|
||||
@@ -358,6 +394,7 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br
|
||||
markSessionUnviewed,
|
||||
markUserMessageSent,
|
||||
resetAllSessionActivityToIdle,
|
||||
interruptBusySessionsAfterRestart,
|
||||
dispose,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -179,6 +179,80 @@ describe('session runtime', () => {
|
||||
expect(runtime.getActiveSessionCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('interrupts busy sessions after restart and broadcasts terminal events once', () => {
|
||||
const events = [];
|
||||
const runtime = createSessionRuntime({
|
||||
writeSseEvent() {},
|
||||
getNotificationClients: () => new Set(),
|
||||
broadcastEvent: (event) => events.push(event),
|
||||
});
|
||||
runtimes.push(runtime);
|
||||
const status = (sessionID, type) => runtime.processOpenCodeSsePayload({
|
||||
type: 'session.status',
|
||||
properties: { sessionID, status: { type } },
|
||||
});
|
||||
|
||||
status('session-busy-1', 'busy');
|
||||
status('session-busy-2', 'retry');
|
||||
status('session-busy-3', 'busy');
|
||||
status('session-idle', 'idle');
|
||||
expect(runtime.getActiveSessionCount()).toBe(3);
|
||||
events.length = 0;
|
||||
|
||||
expect(runtime.interruptBusySessionsAfterRestart()).toEqual({
|
||||
sessionIds: ['session-busy-1', 'session-busy-2', 'session-busy-3'],
|
||||
});
|
||||
|
||||
expect(runtime.getActiveSessionCount()).toBe(0);
|
||||
expect(runtime.getSessionActivitySnapshot()).toEqual({
|
||||
'session-busy-1': { type: 'idle' },
|
||||
'session-busy-2': { type: 'idle' },
|
||||
'session-busy-3': { type: 'idle' },
|
||||
'session-idle': { type: 'idle' },
|
||||
});
|
||||
expect(runtime.getSessionStateSnapshot()).toEqual({
|
||||
'session-busy-1': expect.objectContaining({
|
||||
status: 'idle',
|
||||
metadata: expect.objectContaining({
|
||||
message: 'Interrupted by OpenCode restart',
|
||||
reason: 'opencode-restart',
|
||||
}),
|
||||
}),
|
||||
'session-busy-2': expect.objectContaining({ status: 'idle' }),
|
||||
'session-busy-3': expect.objectContaining({ status: 'idle' }),
|
||||
'session-idle': expect.objectContaining({ status: 'idle' }),
|
||||
});
|
||||
|
||||
const terminalEvents = events.filter((event) => (
|
||||
event.type === 'openchamber:session-status' || event.type === 'session.error'
|
||||
));
|
||||
expect(terminalEvents).toHaveLength(6);
|
||||
for (const sessionId of ['session-busy-1', 'session-busy-2', 'session-busy-3']) {
|
||||
expect(terminalEvents).toContainEqual({
|
||||
type: 'openchamber:session-status',
|
||||
properties: expect.objectContaining({
|
||||
sessionID: sessionId,
|
||||
status: 'idle',
|
||||
}),
|
||||
});
|
||||
expect(terminalEvents).toContainEqual({
|
||||
type: 'session.error',
|
||||
properties: {
|
||||
sessionID: sessionId,
|
||||
error: {
|
||||
name: 'MessageAbortedError',
|
||||
message: 'The running turn was interrupted when OpenCode restarted.',
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(terminalEvents.some((event) => event.properties.sessionID === 'session-idle')).toBe(false);
|
||||
|
||||
events.length = 0;
|
||||
expect(runtime.interruptBusySessionsAfterRestart()).toEqual({ sessionIds: [] });
|
||||
expect(events).toEqual([]);
|
||||
});
|
||||
|
||||
it('restores activity when busy interrupts cooldown without timer underflow', () => {
|
||||
vi.useFakeTimers();
|
||||
const runtime = createSessionRuntime({
|
||||
|
||||
Reference in New Issue
Block a user