Merge main

This commit is contained in:
Bohdan Triapitsyn
2026-08-28 01:27:32 +03:00
2169 changed files with 270795 additions and 76177 deletions
+403 -51
View File
@@ -1,6 +1,9 @@
import { spawn, spawnSync } from 'node:child_process';
import net from 'node:net';
import { stripAppImageArgv0Leak } from '../inherited-env.js';
import { registerManagedProcess, unregisterManagedProcess, reapOrphanedProcesses } from './managed-process-registry.js';
import { applyProviderEnvAliases } from './provider-env-aliases.js';
import { recordStartupPerformance } from './startup-performance.js';
const parsePositiveInt = (value, fallback) => {
const parsed = Number.parseInt(String(value ?? ''), 10);
@@ -15,6 +18,69 @@ const HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES = parsePositiveInt(
const HEALTH_CHECK_INTERVAL_OVERRIDE_MS = parsePositiveInt(process.env.OPENCHAMBER_OPENCODE_HEALTH_INTERVAL_MS, 0);
const HEALTH_CHECK_RESULT_CACHE_MS = parsePositiveInt(process.env.OPENCHAMBER_OPENCODE_HEALTH_CACHE_MS, 750);
const OPENCODE_HEALTH_PATH = '/global/health';
// Last-used directory plus the three most recently opened projects — deeper
// tails are unlikely to be the user's first click and just add background work.
const WARMUP_DIRECTORY_LIMIT = 4;
const WARMUP_REQUEST_TIMEOUT_MS = 30000;
const MANAGED_STDERR_TAIL_MAX_BYTES = 32 * 1024;
const HEALTH_FAILURE_DETAIL_MAX_LENGTH = 256;
const getBoundedTextTail = (value, maxBytes) => {
const buffer = Buffer.from(String(value ?? ''));
if (buffer.byteLength <= maxBytes) return buffer.toString();
return buffer.subarray(buffer.byteLength - maxBytes).toString();
};
const sanitizeDiagnosticText = (value) => String(value ?? '')
.replace(/(https?:\/\/)[^/\s:@]+:[^/\s@]+@/gi, '$1[redacted]@')
.replace(/\b(Bearer)\s+[^\s,;]+/gi, '$1 [redacted]')
// Unquoted `Authorization: <scheme> <credential>` values must be handled
// before the generic key/value rule below: that rule stops at whitespace, so
// it would redact only the scheme word and leave the credential intact.
// Scoped to authorization-style keys so ordinary prose using "basic" or
// "token" is not mangled.
.replace(
/(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}authorization[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*(?:"|')?(?:basic|bearer|token)\s+)[^\s,;"']+/gim,
'$1$2[redacted]',
)
.replace(/([?&][^=&#\s]*(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[^=&#\s]*=)[^&#\s]+/gi, '$1[redacted]')
.replace(
/(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;]+)/gim,
'$1$2[redacted]',
);
const getHealthFailureDetail = (error) => {
const name = String(error?.name || 'Error');
const message = String(error?.message || error || 'Unknown error');
return sanitizeDiagnosticText(`${name}: ${message}`).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH);
};
const classifyHealthProbeError = (error) => {
const name = String(error?.name || '');
const code = String(error?.code || '').toUpperCase();
const message = String(error?.message || error || '');
const normalizedMessage = message.toLowerCase();
if (
name === 'AbortError'
|| name === 'TimeoutError'
|| normalizedMessage.includes('the operation was aborted')
|| normalizedMessage.includes('abortsignal.timeout')
) {
return { class: 'timeout', detail: getHealthFailureDetail(error) };
}
if (code === 'ECONNREFUSED' || normalizedMessage.includes('econnrefused')) {
return { class: 'connection_refused', detail: getHealthFailureDetail(error) };
}
if (
code === 'ECONNRESET'
|| normalizedMessage.includes('econnreset')
|| normalizedMessage.includes('socket hang up')
) {
return { class: 'connection_reset', detail: getHealthFailureDetail(error) };
}
return { class: 'error', detail: getHealthFailureDetail(error) };
};
export const createOpenCodeLifecycleRuntime = (deps) => {
const {
@@ -38,11 +104,53 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
buildAugmentedPath,
buildManagedOpenCodePath,
getManagedOpenCodeShellEnvSnapshot,
getManagedOpenCodeEnv = async () => ({}),
getActiveSessionCount = () => 0,
reapManagedOrphanedProcesses = reapOrphanedProcesses,
getWarmupDirectories = async () => [],
onOpenCodeRestarted = null,
now = Date.now,
} = deps;
const killProcessOnPortWin32 = (port) => {
try {
// Get-NetTCPConnection reads the same locale-independent WinNT API
// netstat's display layer translates (e.g. "LISTENING" renders as
// "ABHÖREN"/"ÉCOUTE"/"ESCUTANDO" on non-English Windows), so this
// works regardless of the OS display language.
const result = spawnSync(
'powershell',
[
'-NoProfile',
'-NonInteractive',
'-Command',
`Get-NetTCPConnection -State Listen -LocalPort ${Number.parseInt(port, 10)} -ErrorAction SilentlyContinue | Select-Object -ExpandProperty OwningProcess`,
],
{ encoding: 'utf8', timeout: 5000, windowsHide: true }
);
const output = result.stdout || '';
const myPid = process.pid;
const pids = new Set();
for (const line of output.split(/\r?\n/)) {
const pid = Number.parseInt(line.trim(), 10);
if (pid && pid !== myPid) pids.add(pid);
}
for (const pid of pids) {
try {
spawnSync('taskkill', ['/PID', String(pid), '/F'], { stdio: 'ignore', timeout: 3000, windowsHide: true });
} catch {
}
}
} catch {
}
};
const killProcessOnPort = (port) => {
if (!port || process.platform === 'win32') return;
if (!port) return;
if (process.platform === 'win32') {
killProcessOnPortWin32(port);
return;
}
try {
const result = spawnSync('lsof', ['-ti', `:${port}`], { encoding: 'utf8', timeout: 5000, windowsHide: true });
const output = result.stdout || '';
@@ -60,7 +168,9 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
}
};
const hasChildProcessExited = (child) => !child || child.exitCode !== null || child.signalCode !== null;
const hasChildProcessExited = (child) => !child
|| (child.exitCode !== null && child.exitCode !== undefined)
|| (child.signalCode !== null && child.signalCode !== undefined);
const isManagedOpenCodeProcessAlive = () => {
const child = state.openCodeProcess;
@@ -74,6 +184,36 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
}
};
const snapshotManagedOpenCodeProcess = (child = state.openCodeProcess) => {
if (!child) return null;
const snapshot = {
pid: child.pid || null,
exitCode: child.exitCode ?? null,
signalCode: child.signalCode ?? null,
stderrTail: getBoundedTextTail(
sanitizeDiagnosticText(child.stderrTail ?? ''),
MANAGED_STDERR_TAIL_MAX_BYTES,
),
};
state.lastManagedOpenCodeProcess = snapshot;
return snapshot;
};
const captureRestartDiagnostics = (reason) => {
const processSnapshot = snapshotManagedOpenCodeProcess();
const diagnostics = {
reason: sanitizeDiagnosticText(String(reason || 'managed-restart')).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH),
healthFailure: state.lastOpenCodeHealthFailure ? { ...state.lastOpenCodeHealthFailure } : null,
process: processSnapshot
? { ...processSnapshot, alive: isManagedOpenCodeProcessAlive() }
: null,
busySessionCount: getActiveSessionCount(),
at: new Date(now()).toISOString(),
};
state.lastOpenCodeRestartDiagnostics = diagnostics;
console.warn('[lifecycle] managed OpenCode restart diagnostics', diagnostics);
};
const waitForChildProcessClose = (child, timeoutMs) => new Promise((resolve) => {
if (!child || hasChildProcessExited(child)) {
resolve(true);
@@ -239,6 +379,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const createManagedOpenCodeServerProcess = async ({ hostname, port, timeout, cwd, env: processEnv, shellEnvKeysCount = 0 }) => {
let binary = (process.env.OPENCODE_BINARY || 'opencode').trim() || 'opencode';
const sourceBinary = binary;
let args = ['serve', '--hostname', hostname, '--port', String(port)];
let launchWrapperType = null;
@@ -262,6 +403,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const pathEntryCount = pathValue ? pathValue.split(process.platform === 'win32' ? ';' : ':').filter(Boolean).length : 0;
state.lastOpenCodeLaunchDiagnostics = {
launchedAt: new Date().toISOString(),
sourceBinary,
binary,
args,
cwd,
@@ -281,6 +423,34 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
windowsHide: true,
stdio: ['ignore', 'pipe', 'pipe'],
});
let runtimeStderrTail = '';
let runtimeStderrAttached = false;
let observedExitCode = null;
let observedSignalCode = null;
const getManagedProcessSnapshot = () => ({
pid: child.pid || null,
exitCode: observedExitCode ?? child.exitCode ?? null,
signalCode: observedSignalCode ?? child.signalCode ?? null,
stderrTail: getBoundedTextTail(sanitizeDiagnosticText(runtimeStderrTail), MANAGED_STDERR_TAIL_MAX_BYTES),
});
const recordManagedProcessExit = (code, signal) => {
if (code !== null && code !== undefined) observedExitCode = code;
if (signal !== null && signal !== undefined) observedSignalCode = signal;
state.lastManagedOpenCodeProcess = getManagedProcessSnapshot();
};
const attachRuntimeStderrCapture = () => {
if (runtimeStderrAttached) return;
runtimeStderrAttached = true;
child.stderr?.on('data', (chunk) => {
runtimeStderrTail = getBoundedTextTail(
`${runtimeStderrTail}${chunk.toString()}`,
MANAGED_STDERR_TAIL_MAX_BYTES,
);
});
};
child.on('exit', recordManagedProcessExit);
child.on('close', recordManagedProcessExit);
const url = await new Promise((resolve, reject) => {
let stdout = '';
@@ -307,6 +477,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
finish(reject, new Error(`Failed to parse server url from output: ${line}`));
return;
}
attachRuntimeStderrCapture();
finish(resolve, match[1]);
return;
}
@@ -354,6 +525,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
return {
url,
pid: child.pid || null,
get exitCode() {
return observedExitCode ?? child.exitCode;
},
get signalCode() {
return observedSignalCode ?? child.signalCode;
},
get stderrTail() {
return getManagedProcessSnapshot().stderrTail;
},
async close() {
await closeManagedOpenCodeChild(child);
},
@@ -394,9 +574,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
});
};
const isOpenCodeProcessHealthy = async () => {
const probeOpenCodeHealthDetailed = async () => {
if (!state.openCodeProcess || !state.openCodePort) {
return false;
return {
healthy: false,
failure: {
class: 'error',
detail: 'Managed OpenCode process or port is unavailable',
},
};
}
try {
@@ -408,14 +594,47 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
},
signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS),
});
if (!response.ok) return false;
const body = await response.json().catch(() => null);
return body?.healthy === true;
} catch {
return false;
if (!response.ok) {
return {
healthy: false,
failure: {
class: 'invalid_response',
detail: `Health endpoint returned HTTP ${response.status ?? 'unknown'}`,
},
};
}
let body;
try {
body = await response.json();
} catch {
return {
healthy: false,
failure: {
class: 'invalid_response',
detail: 'Health endpoint returned invalid JSON',
},
};
}
if (body?.healthy !== true) {
return {
healthy: false,
failure: {
class: 'invalid_response',
detail: 'Health endpoint did not report healthy=true',
},
};
}
return { healthy: true, failure: null };
} catch (error) {
return {
healthy: false,
failure: classifyHealthProbeError(error),
};
}
};
const isOpenCodeProcessHealthy = async () => (await probeOpenCodeHealthDetailed()).healthy;
const probeExternalOpenCode = async (port, origin) => {
if (!port || port <= 0) {
return false;
@@ -462,7 +681,10 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const startOpenCodeOnce = async () => {
const startOpenCodeOnce = async (attempt) => {
const attemptStartedAt = performance.now();
let phaseStartedAt = attemptStartedAt;
recordStartupPerformance('opencode.attempt.start', { attempt });
const desiredPort = env.ENV_CONFIGURED_OPENCODE_PORT ?? 0;
const spawnPort = await resolveManagedOpenCodePort(desiredPort, env.ENV_CONFIGURED_OPENCODE_HOSTNAME);
console.log(
@@ -473,15 +695,29 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
await applyOpencodeBinaryFromSettings({ strict: true });
ensureOpencodeCliEnv();
recordStartupPerformance('opencode.binary.ready', {
attempt,
durationMs: performance.now() - phaseStartedAt,
totalDurationMs: performance.now() - attemptStartedAt,
});
phaseStartedAt = performance.now();
const openCodePassword = await ensureLocalOpenCodeServerPassword({ rotateManaged: true });
const envPath = typeof buildManagedOpenCodePath === 'function'
? buildManagedOpenCodePath()
: typeof buildAugmentedPath === 'function'
? buildAugmentedPath()
: process.env.PATH;
let envPath = process.env.PATH;
if (typeof buildManagedOpenCodePath === 'function') {
envPath = buildManagedOpenCodePath();
} else if (typeof buildAugmentedPath === 'function') {
envPath = buildAugmentedPath();
}
const shellEnv = typeof getManagedOpenCodeShellEnvSnapshot === 'function'
? getManagedOpenCodeShellEnvSnapshot() || {}
: {};
const managedOpenCodeEnv = await getManagedOpenCodeEnv();
recordStartupPerformance('opencode.environment.ready', {
attempt,
durationMs: performance.now() - phaseStartedAt,
totalDurationMs: performance.now() - attemptStartedAt,
});
phaseStartedAt = performance.now();
try {
const serverInstance = await createManagedOpenCodeServerProcess({
@@ -490,17 +726,24 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
timeout: 30000,
cwd: state.openCodeWorkingDirectory,
shellEnvKeysCount: Object.keys(shellEnv).length,
env: {
env: stripAppImageArgv0Leak(applyProviderEnvAliases({
...shellEnv,
...process.env,
...managedOpenCodeEnv,
PATH: envPath,
OPENCODE_SERVER_PASSWORD: openCodePassword,
},
})),
});
if (!serverInstance || !serverInstance.url) {
throw new Error('OpenCode server started but URL is missing');
}
recordStartupPerformance('opencode.process.ready', {
attempt,
durationMs: performance.now() - phaseStartedAt,
totalDurationMs: performance.now() - attemptStartedAt,
});
phaseStartedAt = performance.now();
const url = new URL(serverInstance.url);
const port = parseInt(url.port, 10);
@@ -514,6 +757,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
state.lastOpenCodeError = null;
state.openCodeNotReadySince = 0;
recordStartupPerformance('opencode.health.ready', {
attempt,
durationMs: performance.now() - phaseStartedAt,
totalDurationMs: performance.now() - attemptStartedAt,
outcome: 'ready',
});
return serverInstance;
}
@@ -527,6 +777,11 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
state.lastOpenCodeError = message;
state.openCodePort = null;
syncToHmrState();
recordStartupPerformance('opencode.attempt.error', {
attempt,
totalDurationMs: performance.now() - attemptStartedAt,
outcome: 'error',
});
console.error(`Failed to start OpenCode: ${message}`);
throw error;
}
@@ -536,7 +791,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
let lastError = null;
for (let attempt = 1; attempt <= START_OPEN_CODE_MAX_ATTEMPTS; attempt += 1) {
try {
return await startOpenCodeOnce();
return await startOpenCodeOnce(attempt);
} catch (error) {
lastError = error;
if (error?.code === 'OPENCODE_BINARY_INVALID') {
@@ -559,7 +814,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
throw lastError;
};
const restartOpenCode = async () => {
const restartOpenCode = async (reason = 'managed-restart') => {
if (state.isShuttingDown) return;
if (state.currentRestartPromise) {
await state.currentRestartPromise;
@@ -574,11 +829,12 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
if (state.isExternalOpenCode) {
console.log('Re-probing external OpenCode server...');
const probePort = state.openCodePort || env.ENV_CONFIGURED_OPENCODE_PORT || 4096;
const probePort = state.openCodePort ?? env.ENV_EFFECTIVE_PORT ?? 4096;
const probeOrigin = state.openCodeBaseUrl ?? env.ENV_CONFIGURED_OPENCODE_HOST?.origin;
const healthy = await probeExternalOpenCode(probePort, probeOrigin);
if (healthy) {
console.log(`External OpenCode server on port ${probePort} is healthy`);
state.openCodeBaseUrl = probeOrigin ?? null;
setOpenCodePort(probePort);
state.isOpenCodeReady = true;
state.lastOpenCodeError = null;
@@ -597,6 +853,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
return;
}
captureRestartDiagnostics(reason);
const portToKill = state.openCodePort;
if (state.openCodeProcess) {
@@ -638,6 +895,18 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
setupProxy(state.expressApp);
ensureOpenCodeApiPrefix();
}
// The restart may have landed on a NEW port (the old one can remain
// occupied if killProcessOnPort/waitForPortRelease didn't free it in
// time, on any platform). Upstream event readers pinned to the old
// process would keep the UI silent forever, so rebind them to the
// current port. Best effort: a failure here must not fail the restart
// itself.
try {
onOpenCodeRestarted?.();
} catch (error) {
console.warn('Failed to rebind event stream after OpenCode restart:', error?.message ?? error);
}
})();
try {
@@ -645,7 +914,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
} catch (error) {
console.error(`Failed to restart OpenCode: ${error.message}`);
state.lastOpenCodeError = error.message;
if (!env.ENV_CONFIGURED_OPENCODE_PORT) {
if (!env.ENV_EFFECTIVE_PORT) {
state.openCodePort = null;
syncToHmrState();
}
@@ -751,7 +1020,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
clearResolvedOpenCodeBinary();
await applyOpencodeBinaryFromSettings();
await restartOpenCode();
await restartOpenCode(reason || 'config-change');
// A managed OpenCode process is restarted (and thus re-reads config from
// disk) by restartOpenCode(). An external OpenCode server is NOT owned by
@@ -785,12 +1054,20 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
};
const bootstrapOpenCodeAtStartup = async () => {
const bootstrapStartedAt = performance.now();
let bootstrapError = null;
recordStartupPerformance('opencode.bootstrap.start');
try {
// Before doing anything, reap any OpenCode process WE spawned in a prior
// run that was orphaned by a crash/hard-exit. Verified + scoped to our own
// pids, so it never touches a live instance's or the user's own server.
try {
const { reaped } = await reapOrphanedProcesses({ log: (msg) => console.log(msg) });
const orphanReapStartedAt = performance.now();
const { reaped } = await reapManagedOrphanedProcesses({ log: (msg) => console.log(msg) });
recordStartupPerformance('opencode.orphan-reap.ready', {
durationMs: performance.now() - orphanReapStartedAt,
totalDurationMs: performance.now() - bootstrapStartedAt,
});
if (reaped > 0) console.log(`[lifecycle] startup reaped ${reaped} orphaned OpenCode process(es)`);
} catch (error) {
console.warn('[lifecycle] orphan reap failed:', error?.message ?? error);
@@ -819,15 +1096,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
state.lastOpenCodeError = null;
state.openCodeNotReadySince = 0;
syncToHmrState();
} else if (!env.ENV_EFFECTIVE_PORT && await probeExternalOpenCode(4096)) {
console.log('Auto-detected existing OpenCode server on default port 4096');
setOpenCodePort(4096);
state.isOpenCodeReady = true;
state.isExternalOpenCode = true;
state.lastOpenCodeError = null;
state.openCodeNotReadySince = 0;
syncToHmrState();
} else {
// We never auto-attach to an arbitrary pre-existing OpenCode instance.
// Attaching to an external server requires explicit opt-in via env
// (OPENCODE_HOST / OPENCODE_PORT / OPENCODE_SKIP_START), handled by the
// branches above. Without that opt-in we always start our OWN managed
// instance on a freshly-allocated port. A blind probe of the default
// port 4096 used to hijack a user's separately-running OpenCode (e.g.
// the OpenCode desktop app), coupling our lifecycle to theirs and
// breaking init against an unexpected server version/config.
if (env.ENV_EFFECTIVE_PORT) {
console.log(`Using OpenCode port from environment: ${env.ENV_EFFECTIVE_PORT}`);
setOpenCodePort(env.ENV_EFFECTIVE_PORT);
@@ -844,13 +1121,64 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
try {
await waitForOpenCodeReady();
} catch (error) {
bootstrapError = error;
console.error(`OpenCode readiness check failed: ${error.message}`);
}
} catch (error) {
bootstrapError = error;
console.error(`Failed to start OpenCode: ${error.message}`);
console.log('Continuing without OpenCode integration...');
state.lastOpenCodeError = error.message;
}
recordStartupPerformance(
bootstrapError ? 'opencode.bootstrap.error' : 'opencode.bootstrap.ready',
{
totalDurationMs: performance.now() - bootstrapStartedAt,
outcome: bootstrapError ? 'error' : 'ready',
},
);
if (!bootstrapError) {
void warmOpenCodeDirectories();
}
};
// OpenCode initializes each project directory lazily on its first
// directory-scoped request, and that initialization takes seconds on large
// session stores. Without warming, the user's first session open pays it
// interactively (the chat waits on the message fetch until the directory
// finishes initializing). Warm the most recently used directories right
// after readiness so the work overlaps UI startup instead. Sequential and
// best-effort: a failed or slow directory never blocks the others for long,
// and a restart invalidates the pass via the port/readiness guard.
const warmOpenCodeDirectories = async () => {
let directories = [];
try {
directories = await getWarmupDirectories();
} catch {
return;
}
if (!Array.isArray(directories) || directories.length === 0) return;
const warmedPort = state.openCodePort;
for (const directory of directories.slice(0, WARMUP_DIRECTORY_LIMIT)) {
if (typeof directory !== 'string' || !directory) continue;
if (!state.isOpenCodeReady || state.openCodePort !== warmedPort) return;
let timeout = null;
try {
const controller = new AbortController();
timeout = setTimeout(() => controller.abort(), WARMUP_REQUEST_TIMEOUT_MS);
const url = `${buildOpenCodeUrl('/session/status', '')}?directory=${encodeURIComponent(directory)}`;
await fetch(url, {
method: 'GET',
headers: { Accept: 'application/json', ...getOpenCodeAuthHeaders() },
signal: controller.signal,
});
} catch {
// Best-effort — the directory stays lazy and the UI's own request warms it.
} finally {
if (timeout) clearTimeout(timeout);
}
}
};
/**
@@ -867,29 +1195,32 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const STALE_BUSY_GRACE_MS = 2 * 60 * 1000;
let lastUnhealthyWithBusySessionsAt = 0;
let consecutiveHealthFailures = 0;
let lastCountedHealthFailureAt = 0;
let healthProbePromise = null;
let healthCheckCyclePromise = null;
let lastHealthProbeResult = null;
let healthFailureCountIntervalMs = 15_000;
const resetHealthFailureState = () => {
consecutiveHealthFailures = 0;
lastUnhealthyWithBusySessionsAt = 0;
lastCountedHealthFailureAt = 0;
};
const probeOpenCodeHealth = async () => {
const now = Date.now();
if (lastHealthProbeResult && now - lastHealthProbeResult.at < HEALTH_CHECK_RESULT_CACHE_MS) {
return lastHealthProbeResult.healthy;
const checkedAt = now();
if (lastHealthProbeResult && checkedAt - lastHealthProbeResult.at < HEALTH_CHECK_RESULT_CACHE_MS) {
return lastHealthProbeResult;
}
if (healthProbePromise) {
return healthProbePromise;
}
healthProbePromise = isOpenCodeProcessHealthy()
.then((healthy) => {
lastHealthProbeResult = { at: Date.now(), healthy };
return healthy;
healthProbePromise = probeOpenCodeHealthDetailed()
.then((result) => {
lastHealthProbeResult = { at: now(), ...result };
return lastHealthProbeResult;
})
.finally(() => {
healthProbePromise = null;
@@ -902,24 +1233,24 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
const activeCount = getActiveSessionCount();
if (activeCount === 0) {
lastUnhealthyWithBusySessionsAt = 0;
return false;
return { skip: false, staleBusy: false };
}
const now = Date.now();
const checkedAt = now();
if (!lastUnhealthyWithBusySessionsAt) {
lastUnhealthyWithBusySessionsAt = now;
return true;
lastUnhealthyWithBusySessionsAt = checkedAt;
return { skip: true, staleBusy: false };
}
if (now - lastUnhealthyWithBusySessionsAt >= STALE_BUSY_GRACE_MS) {
if (checkedAt - lastUnhealthyWithBusySessionsAt >= STALE_BUSY_GRACE_MS) {
console.warn(
`[lifecycle] OpenCode unhealthy with ${activeCount} busy session(s) for > 2 min — forcing restart`
);
lastUnhealthyWithBusySessionsAt = 0;
return false;
return { skip: false, staleBusy: true };
}
return true;
return { skip: true, staleBusy: false };
};
const runHealthCheckCycle = async (source) => {
@@ -927,25 +1258,45 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
if (healthCheckCyclePromise) return healthCheckCyclePromise;
healthCheckCyclePromise = (async () => {
const healthy = await probeOpenCodeHealth();
if (!healthy) {
const healthResult = await probeOpenCodeHealth();
if (!healthResult.healthy) {
if (!isManagedOpenCodeProcessAlive()) {
console.log(`[lifecycle] ${source} health check: OpenCode process exited, restarting...`);
consecutiveHealthFailures = 0;
lastHealthProbeResult = null;
await restartOpenCode();
await restartOpenCode(`${source}-process-exited`);
return;
}
const checkedAt = now();
if (lastCountedHealthFailureAt && checkedAt - lastCountedHealthFailureAt < healthFailureCountIntervalMs) {
return;
}
lastCountedHealthFailureAt = checkedAt;
consecutiveHealthFailures += 1;
const healthFailure = healthResult.failure || {
class: 'error',
detail: 'Health check failed without diagnostic detail',
};
state.lastOpenCodeHealthFailure = {
class: healthFailure.class,
detail: healthFailure.detail,
at: new Date(checkedAt).toISOString(),
source,
};
console.warn(
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES})`
`[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES}) class=${healthFailure.class}`
);
if (consecutiveHealthFailures < HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES) return;
if (shouldSkipRestartForBusySessions()) return;
const busyDecision = shouldSkipRestartForBusySessions();
if (busyDecision.skip) return;
console.log(`[lifecycle] ${source} health check failure threshold reached, restarting OpenCode...`);
consecutiveHealthFailures = 0;
lastHealthProbeResult = null;
await restartOpenCode();
await restartOpenCode(
busyDecision.staleBusy
? `${source}-stale-busy-health-failure`
: `${source}-health-failure`,
);
} else {
resetHealthFailureState();
}
@@ -970,6 +1321,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => {
}
const effectiveIntervalMs = HEALTH_CHECK_INTERVAL_OVERRIDE_MS || healthCheckIntervalMs;
healthFailureCountIntervalMs = effectiveIntervalMs;
state.healthCheckInterval = setInterval(async () => {
try {