diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c9a9353..65659752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Chat: if OpenCode restarts while a response is still running, the chat now stops with an interrupted state and a notification to continue instead of hanging silently (thanks to @sum117). + ## [1.19.0] - 2026-08-19 - **Settings/Integrations:** a new Integrations settings page lists Claude Code, Command Code, and Cursor plugins with install, update, setup, and remove actions, plus Discord and Telegram Coming soon placeholders. diff --git a/packages/ui/src/lib/i18n/messages/de.ts b/packages/ui/src/lib/i18n/messages/de.ts index c0c19339..35d1cab5 100644 --- a/packages/ui/src/lib/i18n/messages/de.ts +++ b/packages/ui/src/lib/i18n/messages/de.ts @@ -1901,6 +1901,9 @@ export const dict = { 'chat.revert.toast.undo': 'Zurückgesetzt auf {preview}', 'chat.revert.toast.redo': 'Wiederholt', 'chat.revert.toast.restored': 'Alle Nachrichten wiederhergestellt', + 'chat.toast.opencodeRestartInterrupted.title': 'Chat unterbrochen', + 'chat.toast.opencodeRestartInterrupted.description': 'OpenCode wurde neu gestartet, während noch eine Antwort lief. Senden Sie eine Nachricht, um fortzufahren.', + 'chat.toast.opencodeRestartInterrupted.openSession': 'Sitzung öffnen', 'chat.errorBoundary.title': 'Chat-Fehler', 'chat.errorBoundary.description': 'Die Chat-Oberfläche hat einen Fehler festgestellt. Dies könnte auf ein vorübergehendes Netzwerkproblem oder beschädigte Nachrichtendaten zurückzuführen sein.', 'chat.errorBoundary.sessionLabel': 'Sitzung', diff --git a/packages/ui/src/lib/i18n/messages/en.ts b/packages/ui/src/lib/i18n/messages/en.ts index a5443615..a2af1dcc 100644 --- a/packages/ui/src/lib/i18n/messages/en.ts +++ b/packages/ui/src/lib/i18n/messages/en.ts @@ -2060,6 +2060,9 @@ export const dict = { 'chat.revert.toast.undo': 'Reverted to {preview}', 'chat.revert.toast.redo': 'Redone', 'chat.revert.toast.restored': 'Restored all messages', + 'chat.toast.opencodeRestartInterrupted.title': 'Chat interrupted', + 'chat.toast.opencodeRestartInterrupted.description': 'OpenCode restarted while a response was still running. Send a message to continue.', + 'chat.toast.opencodeRestartInterrupted.openSession': 'Open session', 'chat.errorBoundary.title': 'Chat Error', 'chat.errorBoundary.description': 'The chat interface encountered an error. This might be due to a temporary network issue or corrupted message data.', 'chat.errorBoundary.sessionLabel': 'Session', diff --git a/packages/ui/src/lib/i18n/messages/es.ts b/packages/ui/src/lib/i18n/messages/es.ts index 9bda2677..34649bcd 100644 --- a/packages/ui/src/lib/i18n/messages/es.ts +++ b/packages/ui/src/lib/i18n/messages/es.ts @@ -2038,6 +2038,9 @@ export const dict: Record = { "chat.revert.toast.undo": "Revertido a {preview}", "chat.revert.toast.redo": "Rehecho", "chat.revert.toast.restored": "Todos los mensajes restaurados", + "chat.toast.opencodeRestartInterrupted.title": "Conversación interrumpida", + "chat.toast.opencodeRestartInterrupted.description": "OpenCode se reinició mientras aún se estaba generando una respuesta. Envía un mensaje para continuar.", + "chat.toast.opencodeRestartInterrupted.openSession": "Abrir sesión", "chat.errorBoundary.title": "Error en la conversación", "chat.errorBoundary.description": "La interfaz de la conversación encontró un error. Esto podría deberse a un problema de red temporal o a datos de mensaje corruptos.", "chat.errorBoundary.sessionLabel": "Sesión", diff --git a/packages/ui/src/lib/i18n/messages/fr.ts b/packages/ui/src/lib/i18n/messages/fr.ts index d481a420..bde799e2 100644 --- a/packages/ui/src/lib/i18n/messages/fr.ts +++ b/packages/ui/src/lib/i18n/messages/fr.ts @@ -1802,6 +1802,9 @@ export const dict = { 'chat.revert.toast.undo': 'Revenu à {preview}', 'chat.revert.toast.redo': 'Refait', 'chat.revert.toast.restored': 'Restauré tous les messages', + 'chat.toast.opencodeRestartInterrupted.title': 'Discussion interrompue', + 'chat.toast.opencodeRestartInterrupted.description': 'OpenCode a redémarré alors qu’une réponse était encore en cours. Envoyez un message pour continuer.', + 'chat.toast.opencodeRestartInterrupted.openSession': 'Ouvrir la session', 'chat.errorBoundary.title': 'Erreur de discussion', 'chat.errorBoundary.description': 'L\'interface de discussion a rencontré une erreur. Cela peut être dû à un problème de réseau temporaire ou à des données de message corrompues.', 'chat.errorBoundary.sessionLabel': 'Session', diff --git a/packages/ui/src/lib/i18n/messages/ja.ts b/packages/ui/src/lib/i18n/messages/ja.ts index afc6aae3..a18ca8dc 100644 --- a/packages/ui/src/lib/i18n/messages/ja.ts +++ b/packages/ui/src/lib/i18n/messages/ja.ts @@ -2056,6 +2056,9 @@ export const dict: Record = { 'chat.revert.toast.undo': '{preview}に元に戻しました', 'chat.revert.toast.redo': 'やり直しました', 'chat.revert.toast.restored': 'すべてのメッセージを復元しました', + 'chat.toast.opencodeRestartInterrupted.title': 'チャットが中断されました', + 'chat.toast.opencodeRestartInterrupted.description': '応答の生成中に OpenCode が再起動しました。続行するにはメッセージを送信してください。', + 'chat.toast.opencodeRestartInterrupted.openSession': 'セッションを開く', 'chat.errorBoundary.title': 'チャットエラー', 'chat.errorBoundary.description': 'チャットインターフェースでエラーが発生しました。一時的なネットワーク問題または破損したメッセージデータが原因の可能性があります。', 'chat.errorBoundary.sessionLabel': 'セッション', diff --git a/packages/ui/src/lib/i18n/messages/ko.ts b/packages/ui/src/lib/i18n/messages/ko.ts index eb632cdb..170ac4b2 100644 --- a/packages/ui/src/lib/i18n/messages/ko.ts +++ b/packages/ui/src/lib/i18n/messages/ko.ts @@ -2062,6 +2062,9 @@ export const dict: Record = { 'chat.revert.toast.undo': '{preview}(으)로 되돌림', 'chat.revert.toast.redo': '다시 실행', 'chat.revert.toast.restored': '모든 메시지 복원됨', + 'chat.toast.opencodeRestartInterrupted.title': '채팅이 중단되었습니다', + 'chat.toast.opencodeRestartInterrupted.description': '응답이 진행 중인 동안 OpenCode가 다시 시작되었습니다. 계속하려면 메시지를 보내세요.', + 'chat.toast.opencodeRestartInterrupted.openSession': '세션 열기', 'chat.errorBoundary.title': '채팅 오류', 'chat.errorBoundary.description': '채팅 인터페이스에서 오류가 발생했습니다. 일시적인 네트워크 이슈 또는 손상된 메시지 데이터 때문일 수 있습니다.', 'chat.errorBoundary.sessionLabel': '세션', diff --git a/packages/ui/src/lib/i18n/messages/pl.ts b/packages/ui/src/lib/i18n/messages/pl.ts index db34f661..fd0ebd78 100644 --- a/packages/ui/src/lib/i18n/messages/pl.ts +++ b/packages/ui/src/lib/i18n/messages/pl.ts @@ -766,6 +766,9 @@ export const dict: Record = { 'chat.revert.toast.undo': 'Cofnięte do {preview}', 'chat.revert.toast.redo': 'Ponowione', 'chat.revert.toast.restored': 'Przywrócono wszystkie wiadomości', + 'chat.toast.opencodeRestartInterrupted.title': 'Czat został przerwany', + 'chat.toast.opencodeRestartInterrupted.description': 'OpenCode uruchomił się ponownie podczas generowania odpowiedzi. Wyślij wiadomość, aby kontynuować.', + 'chat.toast.opencodeRestartInterrupted.openSession': 'Otwórz sesję', 'chat.errorBoundary.title': 'Błąd Czatu', 'chat.errorBoundary.description': 'Interfejs czatu napotkał błąd. Może to być spowodowane tymczasowym problemem sieciowym lub uszkodzonymi danymi wiadomości.', 'chat.errorBoundary.sessionLabel': 'Sesja', diff --git a/packages/ui/src/lib/i18n/messages/pt-BR.ts b/packages/ui/src/lib/i18n/messages/pt-BR.ts index d97e3a40..93a53b2c 100644 --- a/packages/ui/src/lib/i18n/messages/pt-BR.ts +++ b/packages/ui/src/lib/i18n/messages/pt-BR.ts @@ -2038,6 +2038,9 @@ export const dict: Record = { "chat.revert.toast.undo": "Revertido para {preview}", "chat.revert.toast.redo": "Refeito", "chat.revert.toast.restored": "Todas as mensagens restauradas", + "chat.toast.opencodeRestartInterrupted.title": "Conversa interrompida", + "chat.toast.opencodeRestartInterrupted.description": "O OpenCode foi reiniciado enquanto uma resposta ainda estava em andamento. Envie uma mensagem para continuar.", + "chat.toast.opencodeRestartInterrupted.openSession": "Abrir sessão", "chat.errorBoundary.title": "Erro na conversa", "chat.errorBoundary.description": "A interface da conversa encontrou um erro. Isso pode ter sido causado por um problema temporário de rede ou por dados de mensagem corrompidos.", "chat.errorBoundary.sessionLabel": "Sessão", diff --git a/packages/ui/src/lib/i18n/messages/uk.ts b/packages/ui/src/lib/i18n/messages/uk.ts index f585de04..ce9a0963 100644 --- a/packages/ui/src/lib/i18n/messages/uk.ts +++ b/packages/ui/src/lib/i18n/messages/uk.ts @@ -2038,6 +2038,9 @@ export const dict: Record = { "chat.revert.toast.undo": "Відкочено до {preview}", "chat.revert.toast.redo": "Повторено", "chat.revert.toast.restored": "Всі повідомлення відновлено", + "chat.toast.opencodeRestartInterrupted.title": "Чат перервано", + "chat.toast.opencodeRestartInterrupted.description": "OpenCode перезапустився, поки відповідь ще формувалася. Надішліть повідомлення, щоб продовжити.", + "chat.toast.opencodeRestartInterrupted.openSession": "Відкрити сесію", "chat.errorBoundary.title": "Помилка чату", "chat.errorBoundary.description": "В інтерфейсі чату сталася помилка. Причиною може бути тимчасова проблема з мережею або пошкоджені дані повідомлення.", "chat.errorBoundary.sessionLabel": "Сесія", diff --git a/packages/ui/src/lib/i18n/messages/zh-CN.ts b/packages/ui/src/lib/i18n/messages/zh-CN.ts index 77e7bf55..eb6b7241 100644 --- a/packages/ui/src/lib/i18n/messages/zh-CN.ts +++ b/packages/ui/src/lib/i18n/messages/zh-CN.ts @@ -2026,6 +2026,9 @@ export const dict: Record = { 'chat.revert.toast.undo': '已撤回至 {preview}', 'chat.revert.toast.redo': '已重做', 'chat.revert.toast.restored': '已恢复全部消息', + 'chat.toast.opencodeRestartInterrupted.title': '聊天已中断', + 'chat.toast.opencodeRestartInterrupted.description': 'OpenCode 在回复仍在生成时重启了。发送一条消息以继续。', + 'chat.toast.opencodeRestartInterrupted.openSession': '打开会话', 'chat.errorBoundary.title': '聊天错误', 'chat.errorBoundary.description': '聊天界面发生错误,可能是临时网络问题或消息数据损坏导致。', 'chat.errorBoundary.sessionLabel': '会话', diff --git a/packages/ui/src/lib/i18n/messages/zh-TW.ts b/packages/ui/src/lib/i18n/messages/zh-TW.ts index 349d4769..89ab710d 100644 --- a/packages/ui/src/lib/i18n/messages/zh-TW.ts +++ b/packages/ui/src/lib/i18n/messages/zh-TW.ts @@ -2030,6 +2030,9 @@ export const dict: Record = { 'chat.revert.toast.undo': '已收回至 {preview}', 'chat.revert.toast.redo': '已重做', 'chat.revert.toast.restored': '已恢復全部訊息', + 'chat.toast.opencodeRestartInterrupted.title': '聊天已中斷', + 'chat.toast.opencodeRestartInterrupted.description': 'OpenCode 在回覆仍在產生時重新啟動。傳送訊息以繼續。', + 'chat.toast.opencodeRestartInterrupted.openSession': '開啟會話', 'chat.errorBoundary.title': '聊天錯誤', 'chat.errorBoundary.description': '聊天介面發生錯誤,可能是暫時網路問題或訊息資料損毀導致。', 'chat.errorBoundary.sessionLabel': '會話', diff --git a/packages/ui/src/sync/DOCUMENTATION.md b/packages/ui/src/sync/DOCUMENTATION.md index 3a2bb657..e883d128 100644 --- a/packages/ui/src/sync/DOCUMENTATION.md +++ b/packages/ui/src/sync/DOCUMENTATION.md @@ -209,7 +209,7 @@ Incomplete-session materialization is deduplicated by runtime, directory, and se When `session.idle` or `session.error` settles a session but the trailing assistant message still contains a `pending` or `running` tool, sync refreshes that session tail. This narrowly reconciles a missed terminal tool-part event without refetching normally completed turns or stale tools from older turns. A stale refresh or delayed part event cannot regress a locally observed terminal tool to an active status. -When a session is authoritatively settled — `session.idle`/`session.error` event, or an authoritative status snapshot that lowers a previously busy session — and the trailing assistant message is still *unfinished* (`time.completed` missing) with active tool parts and no pending question/permission, the turn is treated as interrupted (managed OpenCode process died mid-turn; the server never finalizes the parts, see openchamber#2577 / anomalyco/opencode#19023). The active parts are finalized locally as `error`/`Interrupted` with an end time, so tool timers stop and cards render the error state. The mark is gated on an explicit idle status (absent status is "unknown", never judged), never applies while the session is busy (including question/permission waits), and a later terminal event or refresh supersedes it while a stale `running` refresh cannot regress it. +When a session is authoritatively settled — `session.idle`/`session.error` event, or an authoritative status snapshot that lowers a previously busy session — and the trailing assistant message is still *unfinished* (`time.completed` missing) with no pending question/permission, the turn is treated as interrupted (managed OpenCode process died mid-turn; the server never finalizes the message or parts, see openchamber#2577 / anomalyco/opencode#19023). The unfinished assistant message is completed locally with `MessageAbortedError`, including text-only turns and turns whose tools had already finished, so the chat shows a visible interrupted state. Any active parts are also finalized as `error`/`Interrupted` with an end time, so tool timers stop and cards render the error state. The mark is gated on an explicit idle status (absent status is "unknown", never judged), never applies while the session is busy (including question/permission waits), and a later terminal event can supersede it while a stale unfinished refresh cannot regress the locally finalized message or parts. Directory stores also own session-keyed sidecar notification channels for permissions, questions, and message materialization. High-frequency realtime part events annotate the exact session/message before committing, so visible records, user history, renderability, and sidebar permission and question rows are not notified by unrelated sessions. Structural message replacements notify only changed subscribed session buckets; unannotated bulk part replacement conservatively resets active message subscribers so bootstrap, pagination, rollback, and legacy writers cannot leave stale projections. diff --git a/packages/ui/src/sync/__tests__/interrupted-turn-tools.test.ts b/packages/ui/src/sync/__tests__/interrupted-turn-tools.test.ts index 60968ef7..f19fb59b 100644 --- a/packages/ui/src/sync/__tests__/interrupted-turn-tools.test.ts +++ b/packages/ui/src/sync/__tests__/interrupted-turn-tools.test.ts @@ -3,7 +3,7 @@ * process dies mid-turn, the persisted turn never settles — the trailing * assistant message has no time.completed and its tool parts stay running. * Once the session is authoritatively settled, `interruptedTurnToolParts` - * finalizes the orphaned parts locally. + * completes the assistant message as aborted and finalizes orphaned parts. */ import { describe, expect, test } from "bun:test" import type { Message, Part } from "@opencode-ai/sdk/v2/client" @@ -74,10 +74,15 @@ describe("interruptedTurnToolParts (#2577)", () => { const result = interruptedTurnToolParts(store, "ses_1", 5000) expect(result).not.toBeNull() - const part = result!.parts[0] as { state: { status: string; error: string; time: { end: number } } } + const part = result!.parts![0] as { state: { status: string; error: string; time: { end: number } } } expect(part.state.status).toBe("error") expect(part.state.error).toBe("Interrupted") expect(part.state.time.end).toBe(5000) + expect(result!.messages[0]).toEqual({ + ...unfinishedAssistantMessage("msg_1"), + time: { created: 10, completed: 5000 }, + error: { name: "MessageAbortedError", data: { message: "aborted" }, message: "aborted" }, + }) }) test("busy session is never marked (live work)", () => { @@ -137,16 +142,43 @@ describe("interruptedTurnToolParts (#2577)", () => { const result = interruptedTurnToolParts(store, "ses_1", 5000) expect(result).not.toBeNull() - const statuses = result!.parts.map((part) => (part as { state: { status: string } }).state.status) + const statuses = result!.parts!.map((part) => (part as { state: { status: string } }).state.status) expect(statuses).toEqual(["error", "completed", "error"]) }) - test("no active parts → no change", () => { + test("unfinished assistant with no tools is completed as aborted", () => { const store = state({ session_status: { ses_1: { type: "idle" } }, message: { ses_1: [unfinishedAssistantMessage("msg_1")] }, - part: { msg_1: [completedTool("tool_2", "msg_1")] }, + part: {}, + }) + + const result = interruptedTurnToolParts(store, "ses_1", 5000) + expect(result).not.toBeNull() + expect(result!.parts).toBe(undefined) + expect(result!.messages[0]).toEqual({ + ...unfinishedAssistantMessage("msg_1"), + time: { created: 10, completed: 5000 }, + error: { name: "MessageAbortedError", data: { message: "aborted" }, message: "aborted" }, + }) + }) + + test("completed tools are untouched while the unfinished assistant is aborted", () => { + const completed = completedTool("tool_2", "msg_1") + const store = state({ + session_status: { ses_1: { type: "idle" } }, + message: { ses_1: [unfinishedAssistantMessage("msg_1")] }, + part: { msg_1: [completed] }, + }) + + const result = interruptedTurnToolParts(store, "ses_1", 5000) + expect(result).not.toBeNull() + expect(result!.parts).toBe(undefined) + expect(store.part.msg_1[0]).toBe(completed) + expect(result!.messages[0]).toEqual({ + ...unfinishedAssistantMessage("msg_1"), + time: { created: 10, completed: 5000 }, + error: { name: "MessageAbortedError", data: { message: "aborted" }, message: "aborted" }, }) - expect(interruptedTurnToolParts(store, "ses_1")).toBeNull() }) }) diff --git a/packages/ui/src/sync/__tests__/materialization.test.ts b/packages/ui/src/sync/__tests__/materialization.test.ts index 1f72ed70..9cc5a67f 100644 --- a/packages/ui/src/sync/__tests__/materialization.test.ts +++ b/packages/ui/src/sync/__tests__/materialization.test.ts @@ -119,6 +119,31 @@ describe("materializeSessionSnapshots", () => { expect(result.part.msg_1[0]).toBe(livePart) }) + test("preserves a locally aborted assistant message when a stale unfinished snapshot arrives", () => { + const unfinishedMessage = message("msg_1") + if (unfinishedMessage.role !== "assistant") throw new Error("Expected assistant fixture") + const abortedMessage: Message = { + ...unfinishedMessage, + time: { created: 1, completed: 5000 }, + error: { name: "MessageAbortedError", data: { message: "aborted" } }, + } + const staleMessage = message("msg_1") + const state = { + message: { ses_1: [abortedMessage] }, + part: { msg_1: [] }, + } + + const result = materializeSessionSnapshots( + state, + "ses_1", + [{ info: staleMessage, parts: [] }], + ) + + expect(result.message).toBe(state.message) + expect(result.message.ses_1[0]).toBe(abortedMessage) + expect(result.message.ses_1[0]).not.toBe(staleMessage) + }) + test("does not preserve omitted optimistic user text parts beside server snapshot parts", () => { const optimisticPart = { id: "prt_optimistic", messageID: "msg_1", type: "text", text: "Hello" } as Part const serverPart = part("prt_server", "msg_1", "text", "Hello") diff --git a/packages/ui/src/sync/sync-context.tsx b/packages/ui/src/sync/sync-context.tsx index 271d25d7..772138a1 100644 --- a/packages/ui/src/sync/sync-context.tsx +++ b/packages/ui/src/sync/sync-context.tsx @@ -70,6 +70,7 @@ import { getRuntimeLiveStatusSeed, LIVE_STATUS_TTL_MS } from "./runtime-live-mem import { getRuntimeKey } from "@/lib/runtime-switch" import { getRegisteredRuntimeAPIs } from "@/contexts/runtimeAPIRegistry" import { isFilesystemError } from "@/lib/api/files-errors" +import { formatMessage, useI18nStore } from "@/lib/i18n" import { listGlobalSessionPages } from "@/stores/globalSessions" import { areRequestArraysReferentiallyEqual, collectScopedBlockingRequests } from "./scoped-blocking-requests" import { EMPTY_USER_MESSAGE_HISTORY_SNAPSHOT, buildUserMessageHistorySnapshot, type UserMessageHistorySnapshot } from "./user-message-history" @@ -456,6 +457,32 @@ const handleUiNotificationEvent = (payload: Event, fallbackDirectory: string): b } const notification = properties as UiNotificationPayload + const kind = asOptionalString(notification.kind) + const sessionId = asOptionalString(notification.sessionId) + const directory = asOptionalString(notification.directory) + ?? (fallbackDirectory !== "global" ? fallbackDirectory : "") + + if (kind === "opencode-restart-interrupted") { + const dictionary = useI18nStore.getState().dictionary + const title = formatMessage(dictionary, "chat.toast.opencodeRestartInterrupted.title") + const options = { + id: "opencode-restart-interrupted", + description: formatMessage(dictionary, "chat.toast.opencodeRestartInterrupted.description"), + duration: Infinity, + } + if (sessionId && directory) { + toast.info(title, { + ...options, + action: { + label: formatMessage(dictionary, "chat.toast.opencodeRestartInterrupted.openSession"), + onClick: () => openSessionFromToast(sessionId, directory), + }, + }) + } else { + toast.info(title, options) + } + } + if ((notification.desktopNotificationDelivered === true || notification.desktopStdoutActive === true) && getRuntimeKey() === "local") { return true } @@ -469,9 +496,9 @@ const handleUiNotificationEvent = (payload: Event, fallbackDirectory: string): b title: asOptionalString(notification.title), body: asOptionalString(notification.body), tag: asOptionalString(notification.tag), - kind: asOptionalString(notification.kind), - sessionId: asOptionalString(notification.sessionId), - directory: asOptionalString(notification.directory) ?? (fallbackDirectory && fallbackDirectory !== "global" ? fallbackDirectory : undefined), + kind, + sessionId, + directory: directory || undefined, requireHidden: notification.requireHidden === true, }).catch((error) => { console.warn("[notifications] failed to dispatch UI notification", error) @@ -632,15 +659,24 @@ async function resyncDirectorySessionStatuses( if (mode === "authoritative") { applyGlobalSessionStatusSnapshot(directory, nextStatuses, candidateSessionIds) // An authoritative snapshot that settles sessions previously observed - // busy/retry can orphan running tool parts (managed process died - // mid-turn, #2577): finalize them now. The snapshot write above already - // lowered their status to explicit idle, which is the gate the helper - // requires — a session the snapshot reports busy stays untouched. + // busy/retry can leave their trailing assistant message and tool parts + // unfinished (managed process died mid-turn, #2577): finalize them now. + // The snapshot write above already lowered their status to explicit idle, + // which is the gate the helper requires — a session the snapshot reports + // busy stays untouched. for (const sessionId of candidateSessionIds) { const interrupted = interruptedTurnToolParts(store.getState(), sessionId) if (interrupted) { + if (!interrupted.parts) { + store.setState((state) => ({ + message: { ...state.message, [sessionId]: interrupted.messages }, + })) + continue + } + const interruptedParts = interrupted.parts store.setState((state) => ({ - part: { ...state.part, [interrupted.messageID]: interrupted.parts }, + message: { ...state.message, [sessionId]: interrupted.messages }, + part: { ...state.part, [interrupted.messageID]: interruptedParts }, })) } } @@ -1804,18 +1840,32 @@ export function handleEvent( messageID, }) } - // The reducer already wrote the idle/error status into `draft`; mark the - // orphaned tools using the batched state and publish through the batch. + // The reducer already wrote the idle/error status into `draft`; finalize + // the interrupted message and orphaned tools through the same batch. if (sessionID) { const interrupted = interruptedTurnToolParts(state, sessionID) if (interrupted) { - cloneField("part", (value) => ({ ...(value ?? {}) })) - ;(draft as DirectoryStore).part[interrupted.messageID] = interrupted.parts + cloneField("message", (value) => ({ ...value })) + draft.message[sessionID] = interrupted.messages + if (interrupted.parts) { + cloneField("part", (value) => ({ ...(value ?? {}) })) + draft.part[interrupted.messageID] = interrupted.parts + } if (batch) { batch.states.set(store, draft as DirectoryStore) batch.changedStores.add(store) } else { - store.setState({ part: { ...(store.getState().part), [interrupted.messageID]: interrupted.parts } }) + const currentState = store.getState() + if (interrupted.parts) { + store.setState({ + message: { ...currentState.message, [sessionID]: interrupted.messages }, + part: { ...currentState.part, [interrupted.messageID]: interrupted.parts }, + }) + } else { + store.setState({ + message: { ...currentState.message, [sessionID]: interrupted.messages }, + }) + } } } } @@ -1829,29 +1879,31 @@ export function handleEvent( // // A managed OpenCode process can die mid-turn (crash, health-check restart). // The persisted turn then never settles: the trailing assistant message has -// no `time.completed` and its tool parts stay `pending`/`running` forever — -// the server never finalizes them (anomalyco/opencode#19023). The +// no `time.completed`, and any tool parts can stay `pending`/`running` +// forever — the server never finalizes them (anomalyco/opencode#19023). The // settle-triggered tail refresh above refetches the same stale records, so -// the UI would keep running tool timers and "working" styling indefinitely -// (#2577). +// the UI would keep the assistant message unfinished and any tool timers and +// "working" styling active indefinitely (#2577). // // OpenCode keeps a turn's session busy while it is genuinely alive — // including while waiting for a question/permission reply — so once a // session is AUTHORITATIVELY settled (a `session.idle`/`session.error` // event, or an authoritative status snapshot that lowers a previously busy // session) and the trailing assistant message is still unfinished with -// active tool parts and no pending question/permission, the turn is -// definitively interrupted. Finalize the orphaned parts locally as -// `error`/`Interrupted` with an end time — the same shape OpenCode itself -// writes for cancelled tools. A later terminal part event or a refresh that -// carries the true terminal state supersedes the mark; a stale refresh that -// still reports `running` is rejected by the reducer's and the materializer's -// final-status preservation. +// no pending question/permission, the turn is definitively interrupted. +// Complete the assistant message locally with MessageAbortedError and finalize +// any orphaned parts as `error`/`Interrupted` with an end time — the same shape +// OpenCode itself writes for cancelled tools. A later terminal event can +// supersede the mark; a stale refresh cannot regress the locally final state. +type AssistantMessage = Extract +type SdkMessageAbortedError = Extract, { name: "MessageAbortedError" }> +type LocalMessageAbortedError = SdkMessageAbortedError & { message: string } + export function interruptedTurnToolParts( state: DirectoryStore, sessionID: string, now = Date.now(), -): { messageID: string; parts: Part[] } | null { +): { messageID: string; messages: Message[]; parts?: Part[] } | null { if ((state.question?.[sessionID] ?? []).length > 0) return null if ((state.permission?.[sessionID] ?? []).length > 0) return null @@ -1863,37 +1915,62 @@ export function interruptedTurnToolParts( return null } - const messageID = getStaleRunningToolMessageID(state, sessionID) - if (!messageID) return null - const message = (state.message[sessionID] ?? []).find((candidate) => candidate.id === messageID) - if (!message) return null - if (typeof (message as { time?: { completed?: unknown } }).time?.completed === "number") { + const messages = state.message[sessionID] ?? [] + let messageIndex = -1 + for (let index = messages.length - 1; index >= 0; index -= 1) { + const candidate = messages[index] + if (candidate.role === "user") return null + if (candidate.role !== "assistant") continue + messageIndex = index + break + } + if (messageIndex < 0) return null + + const message = messages[messageIndex] + if (message.role !== "assistant") return null + if (message.time.completed !== undefined) { // The turn finished; a missed terminal tool event is the tail refresh's // job, not an interruption. return null } - const current = state.part[messageID] - if (!current) return null + const messageID = message.id + const nextMessages = [...messages] + const error = { + name: "MessageAbortedError", + data: { message: "aborted" }, + message: "aborted", + } satisfies LocalMessageAbortedError + nextMessages[messageIndex] = { + ...message, + time: { ...message.time, completed: now }, + error, + } - let changed = false - const nextParts = current.map((part) => { + let partsChanged = false + const currentParts = state.part[messageID] + const nextParts = currentParts?.map((part) => { if (part.type !== "tool") return part - const partState = (part as { state?: { status?: unknown; time?: { start?: number } } }).state - if (!partState) return part - if (partState.status !== "pending" && partState.status !== "running") return part - changed = true + if (part.state.status !== "pending" && part.state.status !== "running") return part + partsChanged = true + const partTime = "time" in part.state ? part.state.time : undefined + const start = typeof partTime?.start === "number" ? partTime.start : now return { ...part, state: { - ...partState, - status: "error", + ...part.state, + status: "error" as const, error: "Interrupted", - time: { ...(partState.time ?? {}), end: now }, + time: { start, end: now }, }, - } as Part + } }) - return changed ? { messageID, parts: nextParts } : null + + return { + messageID, + messages: nextMessages, + parts: partsChanged ? nextParts : undefined, + } } // --------------------------------------------------------------------------- diff --git a/packages/vscode/CHANGELOG.md b/packages/vscode/CHANGELOG.md index 4c91c676..e4049133 100644 --- a/packages/vscode/CHANGELOG.md +++ b/packages/vscode/CHANGELOG.md @@ -1,3 +1,7 @@ +## [Unreleased] + +- If OpenCode restarts while a response is still running, the chat now stops with an interrupted state and a notification to continue instead of hanging silently (thanks to @sum117). + ## [1.19.0] - 2026-08-19 - **Settings/Integrations:** a new Integrations settings page lists Claude Code, Command Code, and Cursor plugins with install, update, setup, and remove actions, plus Discord and Telegram Coming soon placeholders. diff --git a/packages/web/server/index.js b/packages/web/server/index.js index 7f40d916..1cf37004 100644 --- a/packages/web/server/index.js +++ b/packages/web/server/index.js @@ -530,6 +530,9 @@ let openCodeApiPrefixDetected = true; let openCodeApiDetectionTimer = null; let lastOpenCodeError = null; let lastOpenCodeLaunchDiagnostics = null; +let lastOpenCodeHealthFailure = null; +let lastManagedOpenCodeProcess = null; +let lastOpenCodeRestartDiagnostics = null; let isOpenCodeReady = false; let openCodeNotReadySince = 0; let isExternalOpenCode = false; @@ -1089,6 +1092,9 @@ Object.defineProperties(openCodeLifecycleState, { openCodeApiDetectionTimer: { get: () => openCodeApiDetectionTimer, set: (value) => { openCodeApiDetectionTimer = value; } }, lastOpenCodeError: { get: () => lastOpenCodeError, set: (value) => { lastOpenCodeError = value; } }, lastOpenCodeLaunchDiagnostics: { get: () => lastOpenCodeLaunchDiagnostics, set: (value) => { lastOpenCodeLaunchDiagnostics = value; } }, + lastOpenCodeHealthFailure: { get: () => lastOpenCodeHealthFailure, set: (value) => { lastOpenCodeHealthFailure = value; } }, + lastManagedOpenCodeProcess: { get: () => lastManagedOpenCodeProcess, set: (value) => { lastManagedOpenCodeProcess = value; } }, + lastOpenCodeRestartDiagnostics: { get: () => lastOpenCodeRestartDiagnostics, set: (value) => { lastOpenCodeRestartDiagnostics = value; } }, isOpenCodeReady: { get: () => isOpenCodeReady, set: (value) => { isOpenCodeReady = value; } }, openCodeNotReadySince: { get: () => openCodeNotReadySince, set: (value) => { openCodeNotReadySince = value; } }, isExternalOpenCode: { get: () => isExternalOpenCode, set: (value) => { isExternalOpenCode = value; } }, @@ -1161,6 +1167,23 @@ const openCodeLifecycleRuntime = createOpenCodeLifecycleRuntime({ } catch (error) { console.warn('Failed to rebind message stream after OpenCode restart:', error?.message ?? error); } + try { + const { sessionIds } = sessionRuntime.interruptBusySessionsAfterRestart(); + if (sessionIds.length > 0) { + const multiple = sessionIds.length > 1; + broadcastUiNotification({ + title: multiple ? 'Chats interrupted' : 'Chat interrupted', + body: multiple + ? 'OpenCode restarted during running responses. Send a message in each chat to continue.' + : 'OpenCode restarted during a running response. Send a message to continue.', + tag: 'opencode-restart-interrupted', + kind: 'opencode-restart-interrupted', + sessionId: sessionIds[0], + }); + } + } catch (error) { + console.warn('Failed to reconcile sessions after OpenCode restart:', error?.message ?? error); + } }, getManagedOpenCodeEnv: async () => { const settings = await readSettingsFromDiskMigrated().catch(() => null); @@ -1658,6 +1681,9 @@ async function main(options = {}) { isOpenCodeReady, lastOpenCodeError, lastOpenCodeLaunchDiagnostics, + lastOpenCodeHealthFailure, + lastManagedOpenCodeProcess, + lastOpenCodeRestartDiagnostics, opencodeBinaryResolved: resolvedOpencodeBinary || null, opencodeBinarySource: resolvedOpencodeBinarySource || null, opencodeLaunchBinary: launchSpec?.binary || null, diff --git a/packages/web/server/lib/opencode/DOCUMENTATION.md b/packages/web/server/lib/opencode/DOCUMENTATION.md index de3afd8d..ae4042cc 100644 --- a/packages/web/server/lib/opencode/DOCUMENTATION.md +++ b/packages/web/server/lib/opencode/DOCUMENTATION.md @@ -111,12 +111,13 @@ This module provides OpenCode server integration utilities for the web server ru - `markSessionUnviewed(sessionId, clientId)` - `markUserMessageSent(sessionId)` - `resetAllSessionActivityToIdle()` + - `interruptBusySessionsAfterRestart()`: settles every session whose authoritative status is `busy`/`retry` or whose activity phase is still busy, broadcasts `openchamber:session-status` idle plus an OpenCode-shaped `session.error`, resets leftover activity/cooldowns, and returns the interrupted session IDs in stable order. - `dispose()` The runtime maintains active-session count incrementally from idempotent activity phase transitions. Upstream stall-timeout and lifecycle health checks read it in O(1); the hourly cleanup removes activity phases older than 24 hours without broadcasting synthetic state transitions. Snapshot generation remains reserved for the session-activity API. ## Public exports (lifecycle.js) -- `createOpenCodeLifecycleRuntime(dependencies)`: creates lifecycle runtime for managed/external OpenCode process orchestration. The optional `onOpenCodeRestarted` dependency (default `null`) is fired after a successful managed restart; `index.js` wires it to `messageStreamRuntime.rebindUpstream()` so event-stream readers rebind to the possibly-new port (a restart can land on a new port while an orphaned process keeps the old one, which would otherwise leave the chat UI silent — issue #2638). +- `createOpenCodeLifecycleRuntime(dependencies)`: creates lifecycle runtime for managed/external OpenCode process orchestration. The optional `onOpenCodeRestarted` dependency (default `null`) is fired after a successful managed restart. `index.js` rebinds event-stream readers to the possibly-new port (#2638), then calls `interruptBusySessionsAfterRestart()` and broadcasts one `opencode-restart-interrupted` UI notification when interrupted turns exist (#2943). - Returned API: - `startOpenCode()` - `restartOpenCode()` @@ -147,6 +148,8 @@ macOS `say` voice enumeration starts concurrently with server composition. The s Transport-triggered health checks share the periodic monitor's failure accounting interval. Rapid WS reconnect callbacks therefore cannot exhaust the managed-process restart threshold using one cached unhealthy result; an exited managed process still restarts immediately. +Managed health failures are classified as `timeout`, `connection_refused`, `connection_reset`, `invalid_response`, or `error`. The lifecycle retains the latest counted failure with a bounded detail string and source. Managed process wrappers continue capturing a sanitized, bounded stderr tail after readiness and retain exit code/signal. Before replacing a managed process, lifecycle snapshots the reason, latest health failure, process diagnostics/aliveness, busy-session count, and timestamp into `lastOpenCodeRestartDiagnostics`; successful startup does not clear this snapshot, and `/health` exposes it for post-restart diagnosis without process environment or credentials. + ## Public exports (env-runtime.js) - `createOpenCodeEnvRuntime(dependencies)`: creates runtime that owns OpenCode CLI environment and binary discovery state. - OpenCode CLI resolution order is persisted settings, environment overrides, bundled Desktop CLI when available, PATH, known install locations, then platform shell discovery. diff --git a/packages/web/server/lib/opencode/lifecycle.js b/packages/web/server/lib/opencode/lifecycle.js index 03734316..7f3bd1df 100644 --- a/packages/web/server/lib/opencode/lifecycle.js +++ b/packages/web/server/lib/opencode/lifecycle.js @@ -22,6 +22,65 @@ const OPENCODE_HEALTH_PATH = '/global/health'; // tails are unlikely to be the user's first click and just add background work. const WARMUP_DIRECTORY_LIMIT = 4; const WARMUP_REQUEST_TIMEOUT_MS = 30000; +const MANAGED_STDERR_TAIL_MAX_BYTES = 32 * 1024; +const HEALTH_FAILURE_DETAIL_MAX_LENGTH = 256; + +const getBoundedTextTail = (value, maxBytes) => { + const buffer = Buffer.from(String(value ?? '')); + if (buffer.byteLength <= maxBytes) return buffer.toString(); + return buffer.subarray(buffer.byteLength - maxBytes).toString(); +}; + +const sanitizeDiagnosticText = (value) => String(value ?? '') + .replace(/(https?:\/\/)[^/\s:@]+:[^/\s@]+@/gi, '$1[redacted]@') + .replace(/\b(Bearer)\s+[^\s,;]+/gi, '$1 [redacted]') + // Unquoted `Authorization: ` values must be handled + // before the generic key/value rule below: that rule stops at whitespace, so + // it would redact only the scheme word and leave the credential intact. + // Scoped to authorization-style keys so ordinary prose using "basic" or + // "token" is not mangled. + .replace( + /(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}authorization[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*(?:"|')?(?:basic|bearer|token)\s+)[^\s,;"']+/gim, + '$1$2[redacted]', + ) + .replace(/([?&][^=&#\s]*(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[^=&#\s]*=)[^&#\s]+/gi, '$1[redacted]') + .replace( + /(^|[\s,{\[])((?:"|')?[a-z0-9_.-]{0,80}(?:token|api[_-]?key|password|secret|authorization|credential|private[_-]?key)[a-z0-9_.-]{0,80}(?:"|')?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;]+)/gim, + '$1$2[redacted]', + ); + +const getHealthFailureDetail = (error) => { + const name = String(error?.name || 'Error'); + const message = String(error?.message || error || 'Unknown error'); + return sanitizeDiagnosticText(`${name}: ${message}`).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH); +}; + +const classifyHealthProbeError = (error) => { + const name = String(error?.name || ''); + const code = String(error?.code || '').toUpperCase(); + const message = String(error?.message || error || ''); + const normalizedMessage = message.toLowerCase(); + + if ( + name === 'AbortError' + || name === 'TimeoutError' + || normalizedMessage.includes('the operation was aborted') + || normalizedMessage.includes('abortsignal.timeout') + ) { + return { class: 'timeout', detail: getHealthFailureDetail(error) }; + } + if (code === 'ECONNREFUSED' || normalizedMessage.includes('econnrefused')) { + return { class: 'connection_refused', detail: getHealthFailureDetail(error) }; + } + if ( + code === 'ECONNRESET' + || normalizedMessage.includes('econnreset') + || normalizedMessage.includes('socket hang up') + ) { + return { class: 'connection_reset', detail: getHealthFailureDetail(error) }; + } + return { class: 'error', detail: getHealthFailureDetail(error) }; +}; export const createOpenCodeLifecycleRuntime = (deps) => { const { @@ -88,6 +147,36 @@ export const createOpenCodeLifecycleRuntime = (deps) => { } }; + const snapshotManagedOpenCodeProcess = (child = state.openCodeProcess) => { + if (!child) return null; + const snapshot = { + pid: child.pid || null, + exitCode: child.exitCode ?? null, + signalCode: child.signalCode ?? null, + stderrTail: getBoundedTextTail( + sanitizeDiagnosticText(child.stderrTail ?? ''), + MANAGED_STDERR_TAIL_MAX_BYTES, + ), + }; + state.lastManagedOpenCodeProcess = snapshot; + return snapshot; + }; + + const captureRestartDiagnostics = (reason) => { + const processSnapshot = snapshotManagedOpenCodeProcess(); + const diagnostics = { + reason: sanitizeDiagnosticText(String(reason || 'managed-restart')).slice(0, HEALTH_FAILURE_DETAIL_MAX_LENGTH), + healthFailure: state.lastOpenCodeHealthFailure ? { ...state.lastOpenCodeHealthFailure } : null, + process: processSnapshot + ? { ...processSnapshot, alive: isManagedOpenCodeProcessAlive() } + : null, + busySessionCount: getActiveSessionCount(), + at: new Date(now()).toISOString(), + }; + state.lastOpenCodeRestartDiagnostics = diagnostics; + console.warn('[lifecycle] managed OpenCode restart diagnostics', diagnostics); + }; + const waitForChildProcessClose = (child, timeoutMs) => new Promise((resolve) => { if (!child || hasChildProcessExited(child)) { resolve(true); @@ -297,6 +386,34 @@ export const createOpenCodeLifecycleRuntime = (deps) => { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'], }); + let runtimeStderrTail = ''; + let runtimeStderrAttached = false; + let observedExitCode = null; + let observedSignalCode = null; + + const getManagedProcessSnapshot = () => ({ + pid: child.pid || null, + exitCode: observedExitCode ?? child.exitCode ?? null, + signalCode: observedSignalCode ?? child.signalCode ?? null, + stderrTail: getBoundedTextTail(sanitizeDiagnosticText(runtimeStderrTail), MANAGED_STDERR_TAIL_MAX_BYTES), + }); + const recordManagedProcessExit = (code, signal) => { + if (code !== null && code !== undefined) observedExitCode = code; + if (signal !== null && signal !== undefined) observedSignalCode = signal; + state.lastManagedOpenCodeProcess = getManagedProcessSnapshot(); + }; + const attachRuntimeStderrCapture = () => { + if (runtimeStderrAttached) return; + runtimeStderrAttached = true; + child.stderr?.on('data', (chunk) => { + runtimeStderrTail = getBoundedTextTail( + `${runtimeStderrTail}${chunk.toString()}`, + MANAGED_STDERR_TAIL_MAX_BYTES, + ); + }); + }; + child.on('exit', recordManagedProcessExit); + child.on('close', recordManagedProcessExit); const url = await new Promise((resolve, reject) => { let stdout = ''; @@ -323,6 +440,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => { finish(reject, new Error(`Failed to parse server url from output: ${line}`)); return; } + attachRuntimeStderrCapture(); finish(resolve, match[1]); return; } @@ -371,10 +489,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => { url, pid: child.pid || null, get exitCode() { - return child.exitCode; + return observedExitCode ?? child.exitCode; }, get signalCode() { - return child.signalCode; + return observedSignalCode ?? child.signalCode; + }, + get stderrTail() { + return getManagedProcessSnapshot().stderrTail; }, async close() { await closeManagedOpenCodeChild(child); @@ -416,9 +537,15 @@ export const createOpenCodeLifecycleRuntime = (deps) => { }); }; - const isOpenCodeProcessHealthy = async () => { + const probeOpenCodeHealthDetailed = async () => { if (!state.openCodeProcess || !state.openCodePort) { - return false; + return { + healthy: false, + failure: { + class: 'error', + detail: 'Managed OpenCode process or port is unavailable', + }, + }; } try { @@ -430,14 +557,47 @@ export const createOpenCodeLifecycleRuntime = (deps) => { }, signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS), }); - if (!response.ok) return false; - const body = await response.json().catch(() => null); - return body?.healthy === true; - } catch { - return false; + if (!response.ok) { + return { + healthy: false, + failure: { + class: 'invalid_response', + detail: `Health endpoint returned HTTP ${response.status ?? 'unknown'}`, + }, + }; + } + let body; + try { + body = await response.json(); + } catch { + return { + healthy: false, + failure: { + class: 'invalid_response', + detail: 'Health endpoint returned invalid JSON', + }, + }; + } + if (body?.healthy !== true) { + return { + healthy: false, + failure: { + class: 'invalid_response', + detail: 'Health endpoint did not report healthy=true', + }, + }; + } + return { healthy: true, failure: null }; + } catch (error) { + return { + healthy: false, + failure: classifyHealthProbeError(error), + }; } }; + const isOpenCodeProcessHealthy = async () => (await probeOpenCodeHealthDetailed()).healthy; + const probeExternalOpenCode = async (port, origin) => { if (!port || port <= 0) { return false; @@ -617,7 +777,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => { throw lastError; }; - const restartOpenCode = async () => { + const restartOpenCode = async (reason = 'managed-restart') => { if (state.isShuttingDown) return; if (state.currentRestartPromise) { await state.currentRestartPromise; @@ -655,6 +815,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => { return; } + captureRestartDiagnostics(reason); const portToKill = state.openCodePort; if (state.openCodeProcess) { @@ -820,7 +981,7 @@ export const createOpenCodeLifecycleRuntime = (deps) => { clearResolvedOpenCodeBinary(); await applyOpencodeBinaryFromSettings(); - await restartOpenCode(); + await restartOpenCode(reason || 'config-change'); // A managed OpenCode process is restarted (and thus re-reads config from // disk) by restartOpenCode(). An external OpenCode server is NOT owned by @@ -1010,17 +1171,17 @@ export const createOpenCodeLifecycleRuntime = (deps) => { const probeOpenCodeHealth = async () => { const checkedAt = now(); if (lastHealthProbeResult && checkedAt - lastHealthProbeResult.at < HEALTH_CHECK_RESULT_CACHE_MS) { - return lastHealthProbeResult.healthy; + return lastHealthProbeResult; } if (healthProbePromise) { return healthProbePromise; } - healthProbePromise = isOpenCodeProcessHealthy() - .then((healthy) => { - lastHealthProbeResult = { at: now(), healthy }; - return healthy; + healthProbePromise = probeOpenCodeHealthDetailed() + .then((result) => { + lastHealthProbeResult = { at: now(), ...result }; + return lastHealthProbeResult; }) .finally(() => { healthProbePromise = null; @@ -1033,13 +1194,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => { const activeCount = getActiveSessionCount(); if (activeCount === 0) { lastUnhealthyWithBusySessionsAt = 0; - return false; + return { skip: false, staleBusy: false }; } const checkedAt = now(); if (!lastUnhealthyWithBusySessionsAt) { lastUnhealthyWithBusySessionsAt = checkedAt; - return true; + return { skip: true, staleBusy: false }; } if (checkedAt - lastUnhealthyWithBusySessionsAt >= STALE_BUSY_GRACE_MS) { @@ -1047,10 +1208,10 @@ export const createOpenCodeLifecycleRuntime = (deps) => { `[lifecycle] OpenCode unhealthy with ${activeCount} busy session(s) for > 2 min — forcing restart` ); lastUnhealthyWithBusySessionsAt = 0; - return false; + return { skip: false, staleBusy: true }; } - return true; + return { skip: true, staleBusy: false }; }; const runHealthCheckCycle = async (source) => { @@ -1058,13 +1219,13 @@ export const createOpenCodeLifecycleRuntime = (deps) => { if (healthCheckCyclePromise) return healthCheckCyclePromise; healthCheckCyclePromise = (async () => { - const healthy = await probeOpenCodeHealth(); - if (!healthy) { + const healthResult = await probeOpenCodeHealth(); + if (!healthResult.healthy) { if (!isManagedOpenCodeProcessAlive()) { console.log(`[lifecycle] ${source} health check: OpenCode process exited, restarting...`); consecutiveHealthFailures = 0; lastHealthProbeResult = null; - await restartOpenCode(); + await restartOpenCode(`${source}-process-exited`); return; } const checkedAt = now(); @@ -1073,15 +1234,30 @@ export const createOpenCodeLifecycleRuntime = (deps) => { } lastCountedHealthFailureAt = checkedAt; consecutiveHealthFailures += 1; + const healthFailure = healthResult.failure || { + class: 'error', + detail: 'Health check failed without diagnostic detail', + }; + state.lastOpenCodeHealthFailure = { + class: healthFailure.class, + detail: healthFailure.detail, + at: new Date(checkedAt).toISOString(), + source, + }; console.warn( - `[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES})` + `[lifecycle] ${source} health check failed (${consecutiveHealthFailures}/${HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES}) class=${healthFailure.class}` ); if (consecutiveHealthFailures < HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES) return; - if (shouldSkipRestartForBusySessions()) return; + const busyDecision = shouldSkipRestartForBusySessions(); + if (busyDecision.skip) return; console.log(`[lifecycle] ${source} health check failure threshold reached, restarting OpenCode...`); consecutiveHealthFailures = 0; lastHealthProbeResult = null; - await restartOpenCode(); + await restartOpenCode( + busyDecision.staleBusy + ? `${source}-stale-busy-health-failure` + : `${source}-health-failure`, + ); } else { resetHealthFailureState(); } diff --git a/packages/web/server/lib/opencode/lifecycle.test.js b/packages/web/server/lib/opencode/lifecycle.test.js index a5a6d39d..20d32e56 100644 --- a/packages/web/server/lib/opencode/lifecycle.test.js +++ b/packages/web/server/lib/opencode/lifecycle.test.js @@ -62,6 +62,9 @@ const createRuntime = (overrides = {}, stateOverrides = {}, envOverrides = {}) = openCodeApiPrefixDetected: false, openCodeApiDetectionTimer: null, lastOpenCodeError: null, + lastOpenCodeHealthFailure: null, + lastManagedOpenCodeProcess: null, + lastOpenCodeRestartDiagnostics: null, isOpenCodeReady: false, openCodeNotReadySince: 0, isExternalOpenCode: false, @@ -75,7 +78,7 @@ const createRuntime = (overrides = {}, stateOverrides = {}, envOverrides = {}) = ...stateOverrides, }; - return createOpenCodeLifecycleRuntime({ + const runtime = createOpenCodeLifecycleRuntime({ state, env: { ENV_CONFIGURED_OPENCODE_PORT: 45678, @@ -111,6 +114,8 @@ const createRuntime = (overrides = {}, stateOverrides = {}, envOverrides = {}) = })), ...overrides, }); + runtime.testState = state; + return runtime; }; describe('OpenCode lifecycle', () => { @@ -234,6 +239,61 @@ describe('OpenCode lifecycle', () => { warn.mockRestore(); }); + it.each([ + { + name: 'timeout', + expectedClass: 'timeout', + fetchResult: () => { + const error = new Error('The operation was aborted'); + error.name = 'AbortError'; + throw error; + }, + }, + { + name: 'connection refusal', + expectedClass: 'connection_refused', + fetchResult: () => { + const error = new Error('connect ECONNREFUSED 127.0.0.1:45678'); + error.code = 'ECONNREFUSED'; + throw error; + }, + }, + { + name: 'invalid JSON', + expectedClass: 'invalid_response', + fetchResult: () => ({ + ok: true, + json: async () => { + throw new SyntaxError('Unexpected token'); + }, + }), + }, + ])('classifies and stores a counted $name health failure', async ({ expectedClass, fetchResult }) => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + globalThis.fetch = vi.fn(fetchResult); + const runtime = createRuntime({}, { + openCodePort: 45678, + openCodeProcess: { + pid: process.pid, + exitCode: null, + signalCode: null, + close: vi.fn(async () => {}), + }, + isOpenCodeReady: true, + }); + + await runtime.triggerHealthCheck(); + + expect(runtime.testState.lastOpenCodeHealthFailure).toEqual({ + class: expectedClass, + detail: expect.any(String), + at: expect.any(String), + source: 'immediate', + }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining(`class=${expectedClass}`)); + warn.mockRestore(); + }); + it('does not mistake a live managed process wrapper for an exited child', async () => { const close = vi.fn(async () => {}); const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -320,6 +380,124 @@ describe('OpenCode lifecycle', () => { expect(onOpenCodeRestarted).toHaveBeenCalledTimes(1); }); + it('retains post-listen stderr and exited process diagnostics across restart', async () => { + const firstChild = createMockChild(); + const replacement = createMockChild(); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + globalThis.fetch = vi.fn(async () => ({ + ok: false, + status: 503, + json: async () => null, + })); + spawnMock.mockImplementationOnce(() => { + queueMicrotask(() => { + firstChild.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n'); + }); + return firstChild; + }); + spawnMock.mockImplementationOnce(() => { + queueMicrotask(() => { + replacement.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n'); + }); + return replacement; + }); + const runtime = createRuntime(); + const server = await runtime.startOpenCode(); + runtime.testState.openCodeProcess = server; + + firstChild.stderr.emit( + 'data', + `${'x'.repeat(40 * 1024)}\ntoken=runtime-secret\nruntime worker failed after startup\n`, + ); + firstChild.exitCode = 7; + firstChild.emit('exit', 7, null); + + expect(server.exitCode).toBe(7); + expect(Buffer.byteLength(server.stderrTail)).toBeLessThanOrEqual(32 * 1024); + expect(server.stderrTail).not.toContain('runtime-secret'); + expect(server.stderrTail).toContain('runtime worker failed after startup'); + + await runtime.triggerHealthCheck(); + + expect(runtime.testState.lastOpenCodeRestartDiagnostics).toEqual({ + reason: 'immediate-process-exited', + healthFailure: null, + process: { + pid: 12345, + exitCode: 7, + signalCode: null, + stderrTail: expect.stringContaining('runtime worker failed after startup'), + alive: false, + }, + busySessionCount: 0, + at: expect.any(String), + }); + expect(runtime.testState.lastManagedOpenCodeProcess).toEqual({ + pid: 12345, + exitCode: 7, + signalCode: null, + stderrTail: expect.stringContaining('runtime worker failed after startup'), + }); + + await runtime.testState.openCodeProcess.close(); + warn.mockRestore(); + }); + + it('redacts Authorization scheme credentials from stderr diagnostics', async () => { + const firstChild = createMockChild(); + const replacement = createMockChild(); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + globalThis.fetch = vi.fn(async () => ({ + ok: false, + status: 503, + json: async () => null, + })); + spawnMock.mockImplementationOnce(() => { + queueMicrotask(() => { + firstChild.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n'); + }); + return firstChild; + }); + spawnMock.mockImplementationOnce(() => { + queueMicrotask(() => { + replacement.stdout.emit('data', 'opencode server listening on http://127.0.0.1:45678\n'); + }); + return replacement; + }); + const runtime = createRuntime(); + const server = await runtime.startOpenCode(); + runtime.testState.openCodeProcess = server; + + firstChild.stderr.emit( + 'data', + 'request rejected: Authorization: Basic dXNlcjpwYXNz\n' + + 'authorization: basic bG93ZXI6Y2FzZQ==\n' + + 'Authorization: Bearer fake-bearer-token-value\n' + + 'falling back to basic health monitor\n' + + 'runtime worker failed after startup\n', + ); + firstChild.exitCode = 7; + firstChild.emit('exit', 7, null); + + expect(server.stderrTail).not.toContain('dXNlcjpwYXNz'); + expect(server.stderrTail).not.toContain('bG93ZXI6Y2FzZQ'); + expect(server.stderrTail).not.toContain('fake-bearer-token-value'); + expect(server.stderrTail).toContain('falling back to basic health monitor'); + expect(server.stderrTail).toContain('runtime worker failed after startup'); + + await runtime.triggerHealthCheck(); + + const diagnosticsTail = runtime.testState.lastOpenCodeRestartDiagnostics.process.stderrTail; + expect(diagnosticsTail).not.toContain('dXNlcjpwYXNz'); + expect(diagnosticsTail).not.toContain('bG93ZXI6Y2FzZQ'); + expect(diagnosticsTail).not.toContain('fake-bearer-token-value'); + expect(diagnosticsTail).toContain('falling back to basic health monitor'); + expect(diagnosticsTail).toContain('runtime worker failed after startup'); + + await runtime.testState.openCodeProcess.close(); + warn.mockRestore(); + }); + it('does not call onOpenCodeRestarted when a managed restart fails', async () => { const close = vi.fn(async () => {}); const onOpenCodeRestarted = vi.fn(); diff --git a/packages/web/server/lib/opencode/restart-session-recovery.test.js b/packages/web/server/lib/opencode/restart-session-recovery.test.js new file mode 100644 index 00000000..013f72cc --- /dev/null +++ b/packages/web/server/lib/opencode/restart-session-recovery.test.js @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createSessionRuntime } from './session-runtime.js'; + +describe('managed OpenCode restart session recovery', () => { + it('settles busy sessions and broadcasts one interruption notification', () => { + const events = []; + const broadcastUiNotification = vi.fn(); + const rebindUpstream = vi.fn(); + const sessionRuntime = createSessionRuntime({ + writeSseEvent() {}, + getNotificationClients: () => new Set(), + broadcastEvent: (event) => events.push(event), + }); + const onOpenCodeRestarted = () => { + rebindUpstream(); + const { sessionIds } = sessionRuntime.interruptBusySessionsAfterRestart(); + if (sessionIds.length > 0) { + const multiple = sessionIds.length > 1; + broadcastUiNotification({ + title: multiple ? 'Chats interrupted' : 'Chat interrupted', + body: multiple + ? 'OpenCode restarted during running responses. Send a message in each chat to continue.' + : 'OpenCode restarted during a running response. Send a message to continue.', + tag: 'opencode-restart-interrupted', + kind: 'opencode-restart-interrupted', + sessionId: sessionIds[0], + }); + } + }; + const markBusy = (sessionID) => sessionRuntime.processOpenCodeSsePayload({ + type: 'session.status', + properties: { sessionID, status: { type: 'busy' } }, + }); + + try { + markBusy('session-1'); + markBusy('session-2'); + markBusy('session-3'); + events.length = 0; + + onOpenCodeRestarted(); + + expect(rebindUpstream).toHaveBeenCalledOnce(); + expect(sessionRuntime.getActiveSessionCount()).toBe(0); + expect(Object.values(sessionRuntime.getSessionStateSnapshot()).map((state) => state.status)) + .toEqual(['idle', 'idle', 'idle']); + expect(events.filter((event) => event.type === 'openchamber:session-status')).toHaveLength(3); + expect(events.filter((event) => event.type === 'session.error')).toHaveLength(3); + expect(broadcastUiNotification).toHaveBeenCalledOnce(); + expect(broadcastUiNotification).toHaveBeenCalledWith(expect.objectContaining({ + kind: 'opencode-restart-interrupted', + sessionId: 'session-1', + })); + } finally { + sessionRuntime.dispose(); + } + }); +}); diff --git a/packages/web/server/lib/opencode/session-runtime.js b/packages/web/server/lib/opencode/session-runtime.js index 071deda3..886daac7 100644 --- a/packages/web/server/lib/opencode/session-runtime.js +++ b/packages/web/server/lib/opencode/session-runtime.js @@ -130,7 +130,8 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br const now = Date.now(); const existing = sessionStates.get(sessionId); const existingAttentionState = sessionAttentionStates.get(sessionId); - if (existing && existing.lastUpdateAt > now - 5000 && status === existing.status) { + const isRestartInterruption = metadata.reason === 'opencode-restart'; + if (existing && existing.lastUpdateAt > now - 5000 && status === existing.status && !isRestartInterruption) { return; } @@ -145,7 +146,7 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br const attentionState = sessionAttentionStates.get(sessionId); const attentionChanged = !!attentionState && existingAttentionState?.needsAttention !== attentionState.needsAttention; const clients = getNotificationClients(); - if (!existing || existing.status !== status || attentionChanged) { + if (!existing || existing.status !== status || attentionChanged || isRestartInterruption) { const state = sessionStates.get(sessionId); const syntheticPayload = { type: 'openchamber:session-status', @@ -293,6 +294,41 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br } }; + const interruptBusySessionsAfterRestart = () => { + const interruptedSessionIds = new Set(); + for (const [sessionId, state] of sessionStates) { + if (state.status === 'busy' || state.status === 'retry') { + interruptedSessionIds.add(sessionId); + } + } + for (const [sessionId, activity] of sessionActivityPhases) { + if (activity.phase === 'busy') { + interruptedSessionIds.add(sessionId); + } + } + + const eventId = `opencode-restart-${Date.now()}`; + for (const sessionId of interruptedSessionIds) { + updateSessionState(sessionId, 'idle', eventId, { + message: 'Interrupted by OpenCode restart', + reason: 'opencode-restart', + }); + broadcastEvent?.({ + type: 'session.error', + properties: { + sessionID: sessionId, + error: { + name: 'MessageAbortedError', + message: 'The running turn was interrupted when OpenCode restarted.', + }, + }, + }); + } + + resetAllSessionActivityToIdle(); + return { sessionIds: [...interruptedSessionIds] }; + }; + const cleanupOldSessionStates = () => { const now = Date.now(); for (const [sessionId, data] of sessionStates) { @@ -358,6 +394,7 @@ export const createSessionRuntime = ({ writeSseEvent, getNotificationClients, br markSessionUnviewed, markUserMessageSent, resetAllSessionActivityToIdle, + interruptBusySessionsAfterRestart, dispose, }; }; diff --git a/packages/web/server/lib/opencode/session-runtime.test.js b/packages/web/server/lib/opencode/session-runtime.test.js index 56e0f02a..2360cd6a 100644 --- a/packages/web/server/lib/opencode/session-runtime.test.js +++ b/packages/web/server/lib/opencode/session-runtime.test.js @@ -179,6 +179,80 @@ describe('session runtime', () => { expect(runtime.getActiveSessionCount()).toBe(0); }); + it('interrupts busy sessions after restart and broadcasts terminal events once', () => { + const events = []; + const runtime = createSessionRuntime({ + writeSseEvent() {}, + getNotificationClients: () => new Set(), + broadcastEvent: (event) => events.push(event), + }); + runtimes.push(runtime); + const status = (sessionID, type) => runtime.processOpenCodeSsePayload({ + type: 'session.status', + properties: { sessionID, status: { type } }, + }); + + status('session-busy-1', 'busy'); + status('session-busy-2', 'retry'); + status('session-busy-3', 'busy'); + status('session-idle', 'idle'); + expect(runtime.getActiveSessionCount()).toBe(3); + events.length = 0; + + expect(runtime.interruptBusySessionsAfterRestart()).toEqual({ + sessionIds: ['session-busy-1', 'session-busy-2', 'session-busy-3'], + }); + + expect(runtime.getActiveSessionCount()).toBe(0); + expect(runtime.getSessionActivitySnapshot()).toEqual({ + 'session-busy-1': { type: 'idle' }, + 'session-busy-2': { type: 'idle' }, + 'session-busy-3': { type: 'idle' }, + 'session-idle': { type: 'idle' }, + }); + expect(runtime.getSessionStateSnapshot()).toEqual({ + 'session-busy-1': expect.objectContaining({ + status: 'idle', + metadata: expect.objectContaining({ + message: 'Interrupted by OpenCode restart', + reason: 'opencode-restart', + }), + }), + 'session-busy-2': expect.objectContaining({ status: 'idle' }), + 'session-busy-3': expect.objectContaining({ status: 'idle' }), + 'session-idle': expect.objectContaining({ status: 'idle' }), + }); + + const terminalEvents = events.filter((event) => ( + event.type === 'openchamber:session-status' || event.type === 'session.error' + )); + expect(terminalEvents).toHaveLength(6); + for (const sessionId of ['session-busy-1', 'session-busy-2', 'session-busy-3']) { + expect(terminalEvents).toContainEqual({ + type: 'openchamber:session-status', + properties: expect.objectContaining({ + sessionID: sessionId, + status: 'idle', + }), + }); + expect(terminalEvents).toContainEqual({ + type: 'session.error', + properties: { + sessionID: sessionId, + error: { + name: 'MessageAbortedError', + message: 'The running turn was interrupted when OpenCode restarted.', + }, + }, + }); + } + expect(terminalEvents.some((event) => event.properties.sessionID === 'session-idle')).toBe(false); + + events.length = 0; + expect(runtime.interruptBusySessionsAfterRestart()).toEqual({ sessionIds: [] }); + expect(events).toEqual([]); + }); + it('restores activity when busy interrupts cooldown without timer underflow', () => { vi.useFakeTimers(); const runtime = createSessionRuntime({