feat(mcp): reconnect failed MCP servers in managed OpenCode

OpenCode marks an MCP server failed when it does not come up at startup or when a live connection drops, and never retries. A new managed plugin reconnects those servers with a per-server backoff (1s doubling to a 30s cap), wakes early on mcp.tools.changed, and stops on dispose. Only the failed state is retried; disabled and auth states stay untouched. The OPENCODE_CONFIG_CONTENT merge that agent-tool and system-prompt each carried is now one shared helper so the three managed plugins compose. Claude-Session: https://claude.ai/code/session_01VqV56Hez25hTxXH4ipJfzH
2026-09-04 22:56:27 +03:00
parent caac24ac3d
commit ce5c0c068e
8 changed files with 375 additions and 45 deletions
@@ -0,0 +1,55 @@
+# Managed MCP Reconnect
+
+## Purpose
+
+OpenCode connects each configured MCP server once, when a project directory is
+first used. A server that does not come up then is marked `failed` and never
+retried; a server whose live connection later drops is marked `failed` too and
+stays that way until OpenCode restarts. This module injects a small plugin into
+the OpenCode process OpenChamber launches that reconnects those servers, so a
+server that was slow to start or crashed mid-session comes back on its own.
+
+## Runtime flow
+
+1. `prepareManagedOpenCodeEnv(configContent)` materializes the plugin under
+   `<openchamber-data-dir>/mcp-reconnect/` and appends its `file://` URL to
+   `OPENCODE_CONFIG_CONTENT` through the shared merge in
+   `packages/web/server/lib/opencode/managed-plugin-config.js`.
+2. It is always on for managed OpenCode. There is no setting, because it only
+   acts on servers OpenCode has already given up on.
+3. OpenCode loads the plugin once per project directory with an SDK client
+   scoped to that directory, so each directory reconnects its own servers.
+4. The plugin reads MCP status one second after load, calls connect for every
+   server in the `failed` state, then re-reads status after a per-server delay
+   that doubles from one second to a cap of thirty. A server seen in any other
+   state resets its counter. While nothing is failed it checks every thirty
+   seconds.
+5. A dropped connection publishes `mcp.tools.changed`, which the plugin uses to
+   check right away instead of waiting out the idle interval.
+6. OpenCode calls the plugin's `dispose` hook when it tears the directory down,
+   which stops the loop.
+
+## Invariants
+
+- Only `failed` is retried. `disabled` is the user's choice, and `needs_auth`
+  or `needs_client_registration` need the user to act; retrying those would
+  either re-enable a server the user turned off or loop on a login prompt.
+- The plugin logs nothing and swallows every error. OpenCode already logs each
+  failed attempt, and a status call failing during an OpenCode restart is not
+  news.
+- One check runs at a time; a wake-up arriving during a check is honored once
+  it finishes rather than starting a second loop.
+
+## What the UI sees
+
+OpenCode publishes no event when a reconnect succeeds. The chat picks the
+server up on the next prompt because tools are resolved from live state, but
+the MCP page reads status on bootstrap and refresh, so it can show `failed` for
+a while after the server is back.
+
+## Runtime parity
+
+- Web and Desktop managed OpenCode: injected automatically.
+- External OpenCode (`OPENCODE_HOST` or skip-start) and VS Code's separate
+  OpenCode lifecycle: not injected, because OpenChamber does not control that
+  process environment.
@@ -0,0 +1,122 @@
+import { pathToFileURL } from 'node:url';
+import { appendManagedPlugin } from '../opencode/managed-plugin-config.js';
+
+/**
+ * OpenCode marks an MCP server `failed` when it does not come up at startup or
+ * when a live connection drops, and never tries again. This plugin runs inside
+ * the managed OpenCode process and reconnects those servers with a per-server
+ * exponential backoff, so a server that was merely slow to start, or a local
+ * one that crashed, comes back without an OpenCode restart.
+ *
+ * Only `failed` is retried. `needs_auth`, `needs_client_registration`, and
+ * `disabled` are user decisions or need user action, and retrying them would
+ * either loop on a login prompt or re-enable a server the user turned off.
+ *
+ * The plugin talks to OpenCode through the SDK client OpenCode hands it, which
+ * is scoped to one project directory, so each open directory reconnects its
+ * own servers. Nothing is logged: OpenCode already reports each failed attempt.
+ */
+const createPluginSource = () => String.raw`
+const INITIAL_RETRY_MS = 1000
+const MAX_RETRY_MS = 30000
+const IDLE_CHECK_MS = 30000
+
+export const OpenChamberMcpReconnectPlugin = async ({ client }) => {
+  let disposed = false
+  let running = false
+  let wakeRequested = false
+  let timer
+  // Consecutive failed attempts per server, cleared once it is seen healthy.
+  const attempts = new Map()
+  const dueAt = new Map()
+
+  const schedule = (delayMs) => {
+    if (disposed) return
+    clearTimeout(timer)
+    timer = setTimeout(tick, delayMs)
+  }
+
+  const tick = async () => {
+    if (running) {
+      wakeRequested = true
+      return
+    }
+    running = true
+    let delay = IDLE_CHECK_MS
+    try {
+      const statuses = (await client.mcp.status())?.data ?? {}
+      const now = Date.now()
+      const due = []
+      for (const [name, entry] of Object.entries(statuses)) {
+        if (entry?.status !== "failed") {
+          attempts.delete(name)
+          dueAt.delete(name)
+          continue
+        }
+        const at = dueAt.get(name) ?? now
+        if (at > now) {
+          delay = Math.min(delay, at - now)
+          continue
+        }
+        due.push(name)
+      }
+      for (const name of attempts.keys()) {
+        if (!Object.hasOwn(statuses, name)) {
+          attempts.delete(name)
+          dueAt.delete(name)
+        }
+      }
+
+      await Promise.allSettled(due.map((name) => client.mcp.connect({ path: { name } })))
+
+      // The result is read on the next tick: a server that came back clears
+      // its counter there, one still failed waits out its backoff.
+      const after = Date.now()
+      for (const name of due) {
+        const count = (attempts.get(name) ?? 0) + 1
+        const wait = Math.min(INITIAL_RETRY_MS * 2 ** (count - 1), MAX_RETRY_MS)
+        attempts.set(name, count)
+        dueAt.set(name, after + wait)
+        delay = Math.min(delay, wait)
+      }
+    } catch {
+      // Status is unavailable while OpenCode is shutting down or restarting;
+      // the idle check picks up again when it is back.
+    } finally {
+      running = false
+      const wake = wakeRequested
+      wakeRequested = false
+      schedule(wake ? INITIAL_RETRY_MS : delay)
+    }
+  }
+
+  schedule(INITIAL_RETRY_MS)
+
+  return {
+    // A dropped connection publishes this event; checking right away beats
+    // waiting out the idle interval.
+    event: async ({ event }) => {
+      if (event?.type === "mcp.tools.changed") schedule(INITIAL_RETRY_MS)
+    },
+    dispose: async () => {
+      disposed = true
+      clearTimeout(timer)
+    },
+  }
+}
+`;
+
+export const createMcpReconnectRuntime = ({ fsPromises, path, dataDir }) => {
+  const pluginDirectory = path.join(dataDir, 'mcp-reconnect');
+  const pluginPath = path.join(pluginDirectory, 'openchamber-mcp-reconnect-plugin.js');
+
+  const prepareManagedOpenCodeEnv = async (rawConfig) => {
+    await fsPromises.mkdir(pluginDirectory, { recursive: true });
+    await fsPromises.writeFile(pluginPath, createPluginSource(), { mode: 0o600 });
+    return {
+      OPENCODE_CONFIG_CONTENT: appendManagedPlugin(rawConfig, pathToFileURL(pluginPath).href, 'MCP reconnect plugin'),
+    };
+  };
+
+  return { prepareManagedOpenCodeEnv };
+};
@@ -0,0 +1,140 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { createMcpReconnectRuntime } from './runtime.js';
+
+const temporaryDirectories = [];
+const disposers = [];
+
+afterEach(async () => {
+  await Promise.all(disposers.splice(0).map((dispose) => dispose()));
+  vi.useRealTimers();
+  await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })));
+});
+
+const materialize = async (rawConfig = '{}') => {
+  const dataDir = await fs.mkdtemp(path.join(os.tmpdir(), 'openchamber-mcp-reconnect-'));
+  temporaryDirectories.push(dataDir);
+  const runtime = createMcpReconnectRuntime({ fsPromises: fs, path, dataDir });
+  const prepared = await runtime.prepareManagedOpenCodeEnv(rawConfig);
+  const pluginPath = path.join(dataDir, 'mcp-reconnect', 'openchamber-mcp-reconnect-plugin.js');
+  const pluginModule = await import(`${pathToFileURL(pluginPath).href}?test=${Date.now()}-${Math.random()}`);
+  return { prepared, pluginPath, plugin: pluginModule.OpenChamberMcpReconnectPlugin };
+};
+
+/**
+ * A stand-in for the SDK client OpenCode hands to plugins. `statuses` is live
+ * state the test mutates; `connect` records when each attempt happened.
+ */
+const createClient = (statuses, { onConnect } = {}) => {
+  const attempts = [];
+  return {
+    attempts,
+    statuses,
+    mcp: {
+      status: vi.fn(async () => ({ data: { ...statuses } })),
+      connect: vi.fn(async ({ path: { name } }) => {
+        attempts.push({ name, at: Date.now() });
+        onConnect?.(name);
+        return { data: true };
+      }),
+    },
+  };
+};
+
+const start = async (plugin, client) => {
+  const hooks = await plugin({ client });
+  disposers.push(hooks.dispose);
+  return hooks;
+};
+
+describe('managed MCP reconnect runtime', () => {
+  it('materializes the plugin and preserves existing plugin entries', async () => {
+    const { prepared, pluginPath } = await materialize('{ "plugin": ["file:///existing.js"], "model": "test/model" }');
+    const config = JSON.parse(prepared.OPENCODE_CONFIG_CONTENT);
+    expect(config.model).toBe('test/model');
+    expect(config.plugin).toEqual(['file:///existing.js', pathToFileURL(pluginPath).href]);
+  });
+
+  it('reconnects only servers in the failed state', async () => {
+    vi.useFakeTimers();
+    const { plugin } = await materialize();
+    const client = createClient({
+      broken: { status: 'failed', error: 'spawn ENOENT' },
+      healthy: { status: 'connected' },
+      off: { status: 'disabled' },
+      login: { status: 'needs_auth' },
+      registration: { status: 'needs_client_registration', error: 'no client id' },
+    });
+    await start(plugin, client);
+
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(client.attempts.map((attempt) => attempt.name)).toEqual(['broken']);
+  });
+
+  it('backs off per server up to a cap while it keeps failing', async () => {
+    vi.useFakeTimers();
+    const { plugin } = await materialize();
+    const client = createClient({ broken: { status: 'failed', error: 'refused' } });
+    const started = Date.now();
+    await start(plugin, client);
+
+    await vi.advanceTimersByTimeAsync(92_000);
+
+    expect(client.attempts.map((attempt) => attempt.at - started)).toEqual([
+      1000, 2000, 4000, 8000, 16_000, 32_000, 62_000, 92_000,
+    ]);
+  });
+
+  it('stops retrying once a server is back and starts fresh when it drops again', async () => {
+    vi.useFakeTimers();
+    const { plugin } = await materialize();
+    const statuses = { flaky: { status: 'failed', error: 'refused' } };
+    const client = createClient(statuses, {
+      onConnect: () => { statuses.flaky = { status: 'connected' }; },
+    });
+    const hooks = await start(plugin, client);
+
+    await vi.advanceTimersByTimeAsync(60_000);
+    expect(client.attempts).toHaveLength(1);
+
+    statuses.flaky = { status: 'failed', error: 'Connection closed' };
+    await hooks.event({ event: { type: 'mcp.tools.changed', properties: { server: 'flaky' } } });
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(client.attempts).toHaveLength(2);
+    expect(client.attempts[1].at - client.attempts[0].at).toBeGreaterThan(30_000);
+  });
+
+  it('keeps going when status is temporarily unavailable', async () => {
+    vi.useFakeTimers();
+    const { plugin } = await materialize();
+    const client = createClient({ broken: { status: 'failed', error: 'refused' } });
+    client.mcp.status.mockRejectedValueOnce(new Error('ECONNREFUSED'));
+    await start(plugin, client);
+
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(client.attempts).toHaveLength(0);
+
+    await vi.advanceTimersByTimeAsync(30_000);
+    expect(client.attempts).toHaveLength(1);
+  });
+
+  it('does nothing after dispose', async () => {
+    vi.useFakeTimers();
+    const { plugin } = await materialize();
+    const client = createClient({ broken: { status: 'failed', error: 'refused' } });
+    const hooks = await plugin({ client });
+
+    await hooks.dispose();
+    await hooks.event({ event: { type: 'mcp.tools.changed', properties: { server: 'broken' } } });
+    await vi.advanceTimersByTimeAsync(120_000);
+
+    expect(client.mcp.status).not.toHaveBeenCalled();
+    expect(client.attempts).toHaveLength(0);
+  });
+});