fix: add concurrency controls for multiple sessions using the same provider (#1069)

* fix: add concurrency controls for multiple sessions using the same provider

Adds OS-inspired scheduling primitives (from HiveMind/AIMD research) to prevent
concurrent sessions from the same provider from experiencing slowdowns, random
stops, and cascading failures.

Server-side:
- Health check skips OpenCode restart when sessions are actively busy — a busy
  server under concurrent load can fail the health check timeout without being
  dead. Staleness guard forces restart if unhealthy+busy persists >2 minutes.
- Upstream SSE stall timeout scaled from 20s to 60s to avoid unnecessary
  reconnections when multiple sessions are waiting for LLM responses.

Client-side (HiveMind primitives, arXiv:2604.17111):
- Transparent retry with exponential backoff (1s→2s→4s, max 32s) for
  429/502/503/504 errors — the #1 most effective primitive from the paper.
- Circuit breaker: opens after 3 consecutive retryable errors, cooldown
  doubles each trip (30s→60s→120s, capped 128s), matching TCP AIMD.
- Per-provider session tracking with TTL eviction (1h idle sweep).
- Fetch-level retry gated on AbortError/TypeError only (not DNS failures).

Refs github-code-review skill findings (all 8 issues resolved).

* fix: use definite assignment assertion for response variable

Fixes TS2454: Variable 'response' is used before being assigned
in strict mode. The for-loop body always assigns it on every path
that reaches the post-loop code, but TS can't prove that.

* fix: add cleanupSession to error paths and remove unreachable code

P1 fixes (Greptile review):
- cleanupSession called on fetch error throw path
- cleanupSession called on non-retryable HTTP error throw path
- Removed unreachable post-loop code (loop always terminates via return or throw)

Adds explicit post-loop throw to satisfy TypeScript strict return check.

* fix: address Greptile review feedback on concurrent session controls

Removes client-side session tracking that leaked on normal completion paths.

The session tracking was redundant — the server-side health check already reads from

sessionRuntime.getSessionActivitySnapshot() for busy-session detection.

Changes:

- Remove activeSessions Set and all session-tracking functions from provider-tracker

- Remove trackSessionStarted/cleanupSession calls from client.ts

- Remove unreachable (response as Response) block after retry loop

- Make upstreamStallTimeoutMs conditional: 60s when >1 sessions, 20s otherwise

Refs #1069

* fix: enforce dynamic concurrency safeguards

---------

Co-authored-by: Bohdan Triapitsyn <artmore@protonmail.com>

packages/ui/src/lib/opencode/client.ts

@@ -15,12 +15,25 @@ import type {
 import type { PermissionRequest } from "@/types/permission";
 import type { QuestionRequest } from "@/types/question";
 import { waitForWorktreeBootstrap } from "@/lib/worktrees/worktreeBootstrap";
+import {
+  assertProviderCircuitClosed,
+  recordProviderSuccess,
+  recordProviderError,
+  shouldRetry,
+  getRetryDelayMs,
+} from "./provider-tracker";
 
 // Use relative path by default (works with both dev and nginx proxy server)
 // Can be overridden with VITE_OPENCODE_URL for absolute URLs in special deployments
 const DEFAULT_BASE_URL = import.meta.env.VITE_OPENCODE_URL || "/api";
 const ABSOLUTE_URL_PATTERN = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//;
 
+const isRetryableFetchError = (error: unknown): boolean => {
+  if (error instanceof DOMException && error.name === 'AbortError') return true;
+  if (error instanceof TypeError) return true;
+  return false;
+};
+
 const ensureAbsoluteBaseUrl = (candidate: string): string => {
   const normalized = typeof candidate === "string" && candidate.trim().length > 0 ? candidate.trim() : "/api";
 
@@ -725,39 +738,58 @@ class OpencodeService {
       });
     }
 
-    let response: Response;
-    try {
-      response = await fetch(url.toString(), {
-        method: 'POST',
-        headers: {
-          'content-type': 'application/json',
-          accept: 'application/json',
-        },
-        body: JSON.stringify({
-          model: {
-            providerID: params.providerID,
-            modelID: params.modelID,
-          },
-          agent: params.agent,
-          variant: params.variant,
-          ...(params.messageId ? { messageID: params.messageId } : {}),
-          ...(params.format ? { format: params.format } : {}),
-          parts,
-        }),
-      });
-    } catch (error) {
-      console.error('[git-generation][browser] prompt_async request failed before response', {
-        sessionId: params.id,
-        url: url.toString(),
-        directory: this.currentDirectory,
-        hasFormat: Boolean(params.format),
-        message: error instanceof Error ? error.message : String(error),
-        error,
-      });
-      throw error;
-    }
+    assertProviderCircuitClosed(params.providerID);
+
+    let response!: Response;
+
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        response = await fetch(url.toString(), {
+          method: 'POST',
+          headers: {
+            'content-type': 'application/json',
+            accept: 'application/json',
+          },
+          body: JSON.stringify({
+            model: {
+              providerID: params.providerID,
+              modelID: params.modelID,
+            },
+            agent: params.agent,
+            variant: params.variant,
+            ...(params.messageId ? { messageID: params.messageId } : {}),
+            ...(params.format ? { format: params.format } : {}),
+            parts,
+          }),
+        });
+      } catch (error) {
+        if (attempt < 2 && isRetryableFetchError(error)) {
+          const delay = getRetryDelayMs(attempt);
+          console.warn(
+            `[prompt] fetch failed for ${params.providerID}/${params.modelID} (attempt ${attempt + 1}/3), retrying in ${delay}ms`,
+            (error as Error)?.message
+          );
+          await new Promise((resolve) => setTimeout(resolve, delay));
+          continue;
+        }
+        recordProviderError(params.providerID);
+        throw error;
+      }
+
+      if (response.ok) {
+        recordProviderSuccess(params.providerID);
+        return tempMessageId;
+      }
+
+      if (shouldRetry(params.providerID, response.status, attempt)) {
+        const delay = getRetryDelayMs(attempt);
+        console.warn(
+          `[prompt] ${response.status} for ${params.providerID}/${params.modelID} (attempt ${attempt + 1}/3), retrying in ${delay}ms`
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
 
-    if (!response.ok) {
       let detail = '';
       try {
         detail = await response.text();
@@ -765,12 +797,13 @@ class OpencodeService {
         // ignore
       }
       const suffix = detail && detail.trim().length > 0 ? `: ${detail.trim()}` : '';
-      throw new Error(`Failed to send message (${response.status})${suffix}`);
+      const error = new Error(`Failed to send message (${response.status})${suffix}`);
+      recordProviderError(params.providerID, response.status);
+      throw error;
     }
-
-    // Return temporary ID for optimistic UI
-    // Real messageID will come from server via SSE events
-    return tempMessageId;
+    // Defensive fallback — all loop paths return/throw, but TypeScript
+    // control flow analysis cannot prove exhaustiveness without this.
+    throw new Error('Failed to send message after retries');
   }
 
   async sendCommand(params: {