openchamber/packages/ui/src/lib/opencode/provider-tracker.ts

/**
 * Provider Circuit-Breaker & Retry Tracker
 *
 * Tracks per-provider error state to enable:
 * - Transparent retry with exponential backoff for transient errors
 * - Circuit breaking (pause requests to a provider during error storms)
 *
 * Inspired by HiveMind (arXiv:2604.17111) OS-inspired scheduling primitives.
 */

const DEFAULT_CIRCUIT_BREAK_THRESHOLD = 3
const DEFAULT_CIRCUIT_COOLDOWN_MS = 30_000
const DEFAULT_RETRY_BASE_DELAY_MS = 1000
const DEFAULT_RETRY_MAX_DELAY_MS = 32_000
const DEFAULT_RETRY_MAX_ATTEMPTS = 3
const PROVIDER_EVICTION_TTL_MS = 60 * 60 * 1000
const PROVIDER_EVICTION_INTERVAL_MS = 10 * 60 * 1000

const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504])

type ProviderState = {
  consecutiveErrors: number
  lastErrorAt: number
  circuitOpen: boolean
  circuitOpenAt: number
  circuitCooldownMs: number
}

const providers = new Map<string, ProviderState>()

function evictStaleProviders(): void {
  const now = Date.now()
  for (const [providerID, state] of providers) {
    if (
      state.consecutiveErrors === 0 &&
      now - state.lastErrorAt > PROVIDER_EVICTION_TTL_MS
    ) {
      providers.delete(providerID)
    }
  }
}

if (typeof setInterval !== 'undefined') {
  const interval = setInterval(evictStaleProviders, PROVIDER_EVICTION_INTERVAL_MS)
  ;(interval as unknown as { unref?: () => void }).unref?.()
}

function getOrCreateProvider(providerID: string): ProviderState {
  let state = providers.get(providerID)
  if (!state) {
    state = {
      consecutiveErrors: 0,
      lastErrorAt: 0,
      circuitOpen: false,
      circuitOpenAt: 0,
      circuitCooldownMs: DEFAULT_CIRCUIT_COOLDOWN_MS,
    }
    providers.set(providerID, state)
  }
  return state
}

export function recordProviderSuccess(providerID: string): void {
  if (!providerID) return
  const state = providers.get(providerID)
  if (!state) return
  state.consecutiveErrors = 0
  state.lastErrorAt = 0
}

export function recordProviderError(providerID: string, status?: number): void {
  if (!providerID) return
  const state = getOrCreateProvider(providerID)
  state.consecutiveErrors += 1
  state.lastErrorAt = Date.now()

  if (
    isCircuitBreakerStatus(status) &&
    state.consecutiveErrors >= DEFAULT_CIRCUIT_BREAK_THRESHOLD
  ) {
    state.circuitOpen = true
    state.circuitOpenAt = Date.now()
    console.warn(
      `[provider-tracker] Circuit opened for ${providerID} after ${state.consecutiveErrors} consecutive errors`
    )
  }
}

function isCircuitBreakerStatus(status?: number): boolean {
  return status !== undefined && RETRYABLE_STATUS_CODES.has(status)
}

function isCircuitOpen(providerID: string): boolean {
  const state = providers.get(providerID)
  if (!state?.circuitOpen) return false

  const elapsed = Date.now() - state.circuitOpenAt
  if (elapsed >= state.circuitCooldownMs) {
    state.circuitOpen = false
    state.consecutiveErrors = 0
    state.circuitCooldownMs = Math.min(
      state.circuitCooldownMs * 2,
      DEFAULT_RETRY_MAX_DELAY_MS * 4
    )
    return false
  }

  return true
}

export function shouldRetry(providerID: string, status: number, attempt: number): boolean {
  if (!RETRYABLE_STATUS_CODES.has(status)) return false
  if (attempt >= DEFAULT_RETRY_MAX_ATTEMPTS - 1) return false
  if (isCircuitOpen(providerID)) return false
  return true
}

export function assertProviderCircuitClosed(providerID: string): void {
  if (!providerID || !isCircuitOpen(providerID)) return
  throw new Error(`Provider ${providerID} is temporarily unavailable after repeated errors. Please retry shortly.`)
}

export function getRetryDelayMs(attempt: number): number {
  const delay = DEFAULT_RETRY_BASE_DELAY_MS * 2 ** attempt
  return Math.min(delay, DEFAULT_RETRY_MAX_DELAY_MS)
}