BuzzbeeSCD/openchamber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make opencode health checks resilient

Browse Source 
Avoids restarting OpenCode after transient health probe failures
Coalesces concurrent health checks and briefly caches probe results
Adds configurable health timeout, retry threshold, interval, and cache settings
This commit is contained in:
Bohdan Triapitsyn
2026-05-18 21:17:02 +03:00
parent eb3c78e224
commit d928185640
1 changed files with 91 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/server/lib/opencode/lifecycle.js
+91 -17
View File
@@ -1,6 +1,19 @@
import { spawn, spawnSync } from 'node:child_process';
import net from 'node:net';
const parsePositiveInt = (value, fallback) => {
const parsed = Number.parseInt(String(value ?? ''), 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
};
const HEALTH_CHECK_TIMEOUT_MS = parsePositiveInt(process.env.OPENCHAMBER_OPENCODE_HEALTH_TIMEOUT_MS, 5000);
const HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES = parsePositiveInt(
process.env.OPENCHAMBER_OPENCODE_HEALTH_CONSECUTIVE_FAILURES,
20
);
const HEALTH_CHECK_INTERVAL_OVERRIDE_MS = parsePositiveInt(process.env.OPENCHAMBER_OPENCODE_HEALTH_INTERVAL_MS, 0);
const HEALTH_CHECK_RESULT_CACHE_MS = parsePositiveInt(process.env.OPENCHAMBER_OPENCODE_HEALTH_CACHE_MS, 750);
export const createOpenCodeLifecycleRuntime = (deps) => {
const {
state,
@@ -49,6 +62,18 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const hasChildProcessExited = (child) => !child || child.exitCode !== null || child.signalCode !== null;
const isManagedOpenCodeProcessAlive = () => {
const child = state.openCodeProcess;
if (!child || hasChildProcessExited(child)) return false;
if (!child.pid) return true;
try {
process.kill(child.pid, 0);
return true;
} catch {
return false;
}
};
const waitForChildProcessClose = (child, timeoutMs) => new Promise((resolve) => {
if (!child || hasChildProcessExited(child)) {
resolve(true);
@@ -363,7 +388,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
Accept: 'application/json',
...getOpenCodeAuthHeaders(),
},
signal: AbortSignal.timeout(5000),
signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS),
});
if (!response.ok) return false;
const body = await response.json().catch(() => null);
@@ -812,6 +837,37 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
*/
const STALE_BUSY_GRACE_MS = 2 * 60 * 1000;
let lastUnhealthyWithBusySessionsAt = 0;
let consecutiveHealthFailures = 0;
let healthProbePromise = null;
let healthCheckCyclePromise = null;
let lastHealthProbeResult = null;
const resetHealthFailureState = () => {
consecutiveHealthFailures = 0;
lastUnhealthyWithBusySessionsAt = 0;
};
const probeOpenCodeHealth = async () => {
const now = Date.now();
if (lastHealthProbeResult && now - lastHealthProbeResult.at < HEALTH_CHECK_RESULT_CACHE_MS) {
return lastHealthProbeResult.healthy;
}
if (healthProbePromise) {
return healthProbePromise;
}
healthProbePromise = isOpenCodeProcessHealthy()
.then((healthy) => {
lastHealthProbeResult = { at: Date.now(), healthy };
return healthy;
})
.finally(() => {
healthProbePromise = null;
});
return healthProbePromise;
};
const shouldSkipRestartForBusySessions = () => {
const activeCount = getActiveSessionCount();
@@ -837,18 +893,43 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
return true;
};
const triggerHealthCheck = async () => {
const runHealthCheckCycle = async (source) => {
if (!state.openCodeProcess || state.isShuttingDown || state.isRestartingOpenCode) return;
if (healthCheckCyclePromise) return healthCheckCyclePromise;
try {
const healthy = await isOpenCodeProcessHealthy();
healthCheckCyclePromise = (async () => {
const healthy = await probeOpenCodeHealth();
if (!healthy) {
if (!isManagedOpenCodeProcessAlive()) {
console.log(`[lifecycle] ${source} health check: OpenCode process exited, restarting...`);
consecutiveHealthFailures = 0;
lastHealthProbeResult = null;
await restartOpenCode();
return;
}
consecutiveHealthFailures += 1;
console.warn(
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES})`
);
if (consecutiveHealthFailures < HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES) return;
if (shouldSkipRestartForBusySessions()) return;
console.log('[lifecycle] immediate health check: OpenCode not healthy, restarting...');
console.log(`[lifecycle] ${source} health check failure threshold reached, restarting OpenCode...`);
consecutiveHealthFailures = 0;
lastHealthProbeResult = null;
await restartOpenCode();
} else {
lastUnhealthyWithBusySessionsAt = 0;
resetHealthFailureState();
}
})().finally(() => {
healthCheckCyclePromise = null;
});
return healthCheckCyclePromise;
};
const triggerHealthCheck = async () => {
try {
await runHealthCheckCycle('immediate');
} catch (error) {
console.error(`[lifecycle] immediate health check error: ${error.message}`);
}
@@ -859,22 +940,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
clearInterval(state.healthCheckInterval);
}
state.healthCheckInterval = setInterval(async () => {
if (!state.openCodeProcess || state.isShuttingDown || state.isRestartingOpenCode) return;
const effectiveIntervalMs = HEALTH_CHECK_INTERVAL_OVERRIDE_MS || healthCheckIntervalMs;
state.healthCheckInterval = setInterval(async () => {
try {
const healthy = await isOpenCodeProcessHealthy();
if (!healthy) {
if (shouldSkipRestartForBusySessions()) return;
console.log('OpenCode process not running, restarting...');
await restartOpenCode();
} else {
lastUnhealthyWithBusySessionsAt = 0;
}
await runHealthCheckCycle('periodic');
} catch (error) {
console.error(`Health check error: ${error.message}`);
}
}, healthCheckIntervalMs);
}, effectiveIntervalMs);
};
return {