BuzzbeeSCD/openchamber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: settle busy sessions after managed OpenCode restart (#3002)

Browse Source 
* fix: reconcile busy sessions after managed OpenCode restart

Forced health-check restarts previously rebound the event stream without
settling in-flight turns, so sessions stayed busy with no terminal state.
Interrupt those sessions, classify health failures, and retain bounded
process diagnostics for post-restart diagnosis.

Fixes #2943

Co-authored-by: serkraser <serkraser@gmail.com>

* fix: surface interrupted chats after OpenCode restart

Complete unfinished assistant turns as aborted once the session is
authoritatively idle, and show a persistent toast so users can continue
instead of remaining silently stranded.

Fixes #2943

Co-authored-by: serkraser <serkraser@gmail.com>

* fix: redact Basic auth credentials in restart diagnostics

The key/value sanitizer stopped at whitespace, so Authorization: Basic
credentials survived in stderr tails and health snapshots. Redact the
scheme token before that rule runs.

Co-authored-by: serkraser <serkraser@gmail.com>
This commit is contained in:
Serhii Dziupin
2026-08-19 11:53:30 +03:00
committed by GitHub
parent 13a45aa5a5
commit 14d7a0ca9b
24 changed files with 806 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/server/lib/opencode/lifecycle.js
+202 -26
View File
@@ -22,6 +22,65 @@ const OPENCODE_HEALTH_PATH = '/global/health';
// tails are unlikely to be the user's first click and just add background work.
const WARMUP_DIRECTORY_LIMIT = 4;
const WARMUP_REQUEST_TIMEOUT_MS = 30000;
const MANAGED_STDERR_TAIL_MAX_BYTES = 32 * 1024;
const HEALTH_FAILURE_DETAIL_MAX_LENGTH = 256;
const getBoundedTextTail = (value, maxBytes) => {
const buffer = Buffer.from(String(value ?? ''));
if (buffer.byteLength <= maxBytes) return buffer.toString();
return buffer.subarray(buffer.byteLength - maxBytes).toString();
};
const sanitizeDiagnosticText = (value) => String(value ?? '')
.replace(/(https?:\/\/)[^/\s:@]+:[^/\s@]+@/gi, '$1[redacted]@')
.replace(/\b(Bearer)\s+[^\s,;]+/gi, '$1 [redacted]')
// Unquoted `Authorization: <scheme> <credential>` values must be handled
// before the generic key/value rule below: that rule stops at whitespace, so
// it would redact only the scheme word and leave the credential intact.
// Scoped to authorization-style keys so ordinary prose using "basic" or
// "token" is not mangled.
.replace(
/(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}authorization[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*(?:"|')?(?:basic|bearer|token)\s+)[^\s,;"']+/gim,
'$1$2[redacted]',
)
.replace(/([?&][^=&#\s]*(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[^=&#\s]*=)[^&#\s]+/gi, '$1[redacted]')
.replace(
/(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;]+)/gim,
'$1$2[redacted]',
);
const getHealthFailureDetail = (error) => {
const name = String(error?.name || 'Error');
const message = String(error?.message || error || 'Unknown error');
return sanitizeDiagnosticText(`${name}: ${message}`).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH);
};
const classifyHealthProbeError = (error) => {
const name = String(error?.name || '');
const code = String(error?.code || '').toUpperCase();
const message = String(error?.message || error || '');
const normalizedMessage = message.toLowerCase();
if (
name === 'AbortError'
|| name === 'TimeoutError'
|| normalizedMessage.includes('the operation was aborted')
|| normalizedMessage.includes('abortsignal.timeout')
) {
return { class: 'timeout', detail: getHealthFailureDetail(error) };
}
if (code === 'ECONNREFUSED' || normalizedMessage.includes('econnrefused')) {
return { class: 'connection_refused', detail: getHealthFailureDetail(error) };
}
if (
code === 'ECONNRESET'
|| normalizedMessage.includes('econnreset')
|| normalizedMessage.includes('socket hang up')
) {
return { class: 'connection_reset', detail: getHealthFailureDetail(error) };
}
return { class: 'error', detail: getHealthFailureDetail(error) };
};
export const createOpenCodeLifecycleRuntime = (deps) => {
const {
@@ -88,6 +147,36 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
}
};
const snapshotManagedOpenCodeProcess = (child = state.openCodeProcess) => {
if (!child) return null;
const snapshot = {
pid: child.pid || null,
exitCode: child.exitCode ?? null,
signalCode: child.signalCode ?? null,
stderrTail: getBoundedTextTail(
sanitizeDiagnosticText(child.stderrTail ?? ''),
MANAGED_STDERR_TAIL_MAX_BYTES,
),
};
state.lastManagedOpenCodeProcess = snapshot;
return snapshot;
};
const captureRestartDiagnostics = (reason) => {
const processSnapshot = snapshotManagedOpenCodeProcess();
const diagnostics = {
reason: sanitizeDiagnosticText(String(reason || 'managed-restart')).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH),
healthFailure: state.lastOpenCodeHealthFailure ? { ...state.lastOpenCodeHealthFailure } : null,
process: processSnapshot
? { ...processSnapshot, alive: isManagedOpenCodeProcessAlive() }
: null,
busySessionCount: getActiveSessionCount(),
at: new Date(now()).toISOString(),
};
state.lastOpenCodeRestartDiagnostics = diagnostics;
console.warn('[lifecycle] managed OpenCode restart diagnostics', diagnostics);
};
const waitForChildProcessClose = (child, timeoutMs) => new Promise((resolve) => {
if (!child || hasChildProcessExited(child)) {
resolve(true);
@@ -297,6 +386,34 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
windowsHide: true,
stdio: ['ignore', 'pipe', 'pipe'],
});
let runtimeStderrTail = '';
let runtimeStderrAttached = false;
let observedExitCode = null;
let observedSignalCode = null;
const getManagedProcessSnapshot = () => ({
pid: child.pid || null,
exitCode: observedExitCode ?? child.exitCode ?? null,
signalCode: observedSignalCode ?? child.signalCode ?? null,
stderrTail: getBoundedTextTail(sanitizeDiagnosticText(runtimeStderrTail), MANAGED_STDERR_TAIL_MAX_BYTES),
});
const recordManagedProcessExit = (code, signal) => {
if (code !== null && code !== undefined) observedExitCode = code;
if (signal !== null && signal !== undefined) observedSignalCode = signal;
state.lastManagedOpenCodeProcess = getManagedProcessSnapshot();
};
const attachRuntimeStderrCapture = () => {
if (runtimeStderrAttached) return;
runtimeStderrAttached = true;
child.stderr?.on('data', (chunk) => {
runtimeStderrTail = getBoundedTextTail(
`${runtimeStderrTail}${chunk.toString()}`,
MANAGED_STDERR_TAIL_MAX_BYTES,
);
});
};
child.on('exit', recordManagedProcessExit);
child.on('close', recordManagedProcessExit);
const url = await new Promise((resolve, reject) => {
let stdout = '';
@@ -323,6 +440,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
finish(reject, new Error(`Failed to parse server url from output: ${line}`));
return;
}
attachRuntimeStderrCapture();
finish(resolve, match[1]);
return;
}
@@ -371,10 +489,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
url,
pid: child.pid || null,
get exitCode() {
return child.exitCode;
return observedExitCode ?? child.exitCode;
},
get signalCode() {
return child.signalCode;
return observedSignalCode ?? child.signalCode;
},
get stderrTail() {
return getManagedProcessSnapshot().stderrTail;
},
async close() {
await closeManagedOpenCodeChild(child);
@@ -416,9 +537,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
});
};
const isOpenCodeProcessHealthy = async () => {
const probeOpenCodeHealthDetailed = async () => {
if (!state.openCodeProcess || !state.openCodePort) {
return false;
return {
healthy: false,
failure: {
class: 'error',
detail: 'Managed OpenCode process or port is unavailable',
},
};
}
try {
@@ -430,14 +557,47 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
},
signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS),
});
if (!response.ok) return false;
const body = await response.json().catch(() => null);
return body?.healthy === true;
} catch {
return false;
if (!response.ok) {
return {
healthy: false,
failure: {
class: 'invalid_response',
detail: `Health endpoint returned HTTP ${response.status ?? 'unknown'}`,
},
};
}
let body;
try {
body = await response.json();
} catch {
return {
healthy: false,
failure: {
class: 'invalid_response',
detail: 'Health endpoint returned invalid JSON',
},
};
}
if (body?.healthy !== true) {
return {
healthy: false,
failure: {
class: 'invalid_response',
detail: 'Health endpoint did not report healthy=true',
},
};
}
return { healthy: true, failure: null };
} catch (error) {
return {
healthy: false,
failure: classifyHealthProbeError(error),
};
}
};
const isOpenCodeProcessHealthy = async () => (await probeOpenCodeHealthDetailed()).healthy;
const probeExternalOpenCode = async (port, origin) => {
if (!port || port <= 0) {
return false;
@@ -617,7 +777,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
throw lastError;
};
const restartOpenCode = async () => {
const restartOpenCode = async (reason = 'managed-restart') => {
if (state.isShuttingDown) return;
if (state.currentRestartPromise) {
await state.currentRestartPromise;
@@ -655,6 +815,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
return;
}
captureRestartDiagnostics(reason);
const portToKill = state.openCodePort;
if (state.openCodeProcess) {
@@ -820,7 +981,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
clearResolvedOpenCodeBinary();
await applyOpencodeBinaryFromSettings();
await restartOpenCode();
await restartOpenCode(reason || 'config-change');
// A managed OpenCode process is restarted (and thus re-reads config from
// disk) by restartOpenCode(). An external OpenCode server is NOT owned by
@@ -1010,17 +1171,17 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const probeOpenCodeHealth = async () => {
const checkedAt = now();
if (lastHealthProbeResult && checkedAt - lastHealthProbeResult.at < HEALTH_CHECK_RESULT_CACHE_MS) {
return lastHealthProbeResult.healthy;
return lastHealthProbeResult;
}
if (healthProbePromise) {
return healthProbePromise;
}
healthProbePromise = isOpenCodeProcessHealthy()
.then((healthy) => {
lastHealthProbeResult = { at: now(), healthy };
return healthy;
healthProbePromise = probeOpenCodeHealthDetailed()
.then((result) => {
lastHealthProbeResult = { at: now(), ...result };
return lastHealthProbeResult;
})
.finally(() => {
healthProbePromise = null;
@@ -1033,13 +1194,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const activeCount = getActiveSessionCount();
if (activeCount === 0) {
lastUnhealthyWithBusySessionsAt = 0;
return false;
return { skip: false, staleBusy: false };
}
const checkedAt = now();
if (!lastUnhealthyWithBusySessionsAt) {
lastUnhealthyWithBusySessionsAt = checkedAt;
return true;
return { skip: true, staleBusy: false };
}
if (checkedAt - lastUnhealthyWithBusySessionsAt >= STALE_BUSY_GRACE_MS) {
@@ -1047,10 +1208,10 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
`[lifecycle] OpenCode unhealthy with ${activeCount} busy session(s) for > 2 min — forcing restart`
);
lastUnhealthyWithBusySessionsAt = 0;
return false;
return { skip: false, staleBusy: true };
}
return true;
return { skip: true, staleBusy: false };
};
const runHealthCheckCycle = async (source) => {
@@ -1058,13 +1219,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
if (healthCheckCyclePromise) return healthCheckCyclePromise;
healthCheckCyclePromise = (async () => {
const healthy = await probeOpenCodeHealth();
if (!healthy) {
const healthResult = await probeOpenCodeHealth();
if (!healthResult.healthy) {
if (!isManagedOpenCodeProcessAlive()) {
console.log(`[lifecycle] ${source} health check: OpenCode process exited, restarting...`);
consecutiveHealthFailures = 0;
lastHealthProbeResult = null;
await restartOpenCode();
await restartOpenCode(`${source}-process-exited`);
return;
}
const checkedAt = now();
@@ -1073,15 +1234,30 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
}
lastCountedHealthFailureAt = checkedAt;
consecutiveHealthFailures += 1;
const healthFailure = healthResult.failure || {
class: 'error',
detail: 'Health check failed without diagnostic detail',
};
state.lastOpenCodeHealthFailure = {
class: healthFailure.class,
detail: healthFailure.detail,
at: new Date(checkedAt).toISOString(),
source,
};
console.warn(
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES})`
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES}) class=${healthFailure.class}`
);
if (consecutiveHealthFailures < HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES) return;
if (shouldSkipRestartForBusySessions()) return;
const busyDecision = shouldSkipRestartForBusySessions();
if (busyDecision.skip) return;
console.log(`[lifecycle] ${source} health check failure threshold reached, restarting OpenCode...`);
consecutiveHealthFailures = 0;
lastHealthProbeResult = null;
await restartOpenCode();
await restartOpenCode(
busyDecision.staleBusy
? `${source}-stale-busy-health-failure`
: `${source}-health-failure`,
);
} else {
resetHealthFailureState();
}