feat: surface session goal evaluation model and add diagnostics

Shows the evaluation provider/model used for the latest successful goal audit in the UI. Persists evaluation model metadata on session goals and covers it in tests. Adds diagnostic logging for small-model calls and session-goal audit parsing.
2026-07-15 08:56:10 +03:00
parent 4eac90ad44
commit e48a9397f1
17 changed files with 106 additions and 6 deletions
@@ -117,6 +117,14 @@ export function SessionGoalDialog({ open, onOpenChange, sessionId, directory }:
              {goal.statusReason && (goal.status === 'blocked' || goal.status === 'budgetLimited') ? (
                <p className="typography-meta text-muted-foreground/70">{goal.statusReason}</p>
              ) : null}
+              {goal.evaluationProviderID || goal.evaluationModelID ? (
+                <div className="flex items-baseline gap-2 typography-meta">
+                  <span className="text-muted-foreground/70">{t('chat.goal.dialog.evaluationModelLabel')}</span>
+                  <span className="min-w-0 break-all text-foreground">
+                    {[goal.evaluationProviderID, goal.evaluationModelID].filter(Boolean).join('/')}
+                  </span>
+                </div>
+              ) : null}
            </div>
          )}

@@ -1416,6 +1416,7 @@ export const dict = {
  'chat.goal.dialog.objectiveLabel': 'Objective',
  'chat.goal.dialog.objectivePlaceholder': 'Describe the end state the agent should reach and verify…',
  'chat.goal.dialog.budgetLabel': 'Token budget',
+  'chat.goal.dialog.evaluationModelLabel': 'Evaluation model',
  'chat.goal.status.active': 'Active',
  'chat.goal.status.evaluating': 'Evaluating…',
  'chat.goal.status.paused': 'Paused',
@@ -1394,6 +1394,7 @@ export const dict: Record<I18nKey, string> = {
  "chat.goal.dialog.objectiveLabel": "Objetivo",
  "chat.goal.dialog.objectivePlaceholder": "Describe el estado final que el agente debe alcanzar y verificar…",
  "chat.goal.dialog.budgetLabel": "Presupuesto de tokens",
+  "chat.goal.dialog.evaluationModelLabel": "Modelo de evaluación",
  "chat.goal.status.active": "Activo",
  "chat.goal.status.evaluating": "Evaluando…",
  "chat.goal.status.paused": "En pausa",
@@ -1236,6 +1236,7 @@ export const dict = {
  'chat.goal.dialog.objectiveLabel': 'Objectif',
  'chat.goal.dialog.objectivePlaceholder': 'Décrivez l\'état final que l\'agent doit atteindre et vérifier…',
  'chat.goal.dialog.budgetLabel': 'Budget de tokens',
+  'chat.goal.dialog.evaluationModelLabel': 'Modèle d’évaluation',
  'chat.goal.status.active': 'Actif',
  'chat.goal.status.evaluating': 'Évaluation…',
  'chat.goal.status.paused': 'En pause',
@@ -1412,6 +1412,7 @@ export const dict: Record<I18nKey, string> = {
  'chat.goal.dialog.objectiveLabel': '目標',
  'chat.goal.dialog.objectivePlaceholder': 'エージェントが到達して検証すべき最終状態を記述してください…',
  'chat.goal.dialog.budgetLabel': 'トークン予算',
+  'chat.goal.dialog.evaluationModelLabel': '評価モデル',
  'chat.goal.status.active': '進行中',
  'chat.goal.status.evaluating': '評価中…',
  'chat.goal.status.paused': '一時停止',
@@ -1418,6 +1418,7 @@ export const dict: Record<I18nKey, string> = {
  'chat.goal.dialog.objectiveLabel': '목표',
  'chat.goal.dialog.objectivePlaceholder': '에이전트가 도달하고 검증해야 할 최종 상태를 설명하세요…',
  'chat.goal.dialog.budgetLabel': '토큰 예산',
+  'chat.goal.dialog.evaluationModelLabel': '평가 모델',
  'chat.goal.status.active': '진행 중',
  'chat.goal.status.evaluating': '평가 중…',
  'chat.goal.status.paused': '일시 중지됨',
@@ -2104,6 +2104,7 @@ export const dict: Record<I18nKey, string> = {
  'chat.goal.dialog.objectiveLabel': 'Cel',
  'chat.goal.dialog.objectivePlaceholder': 'Opisz stan końcowy, który agent ma osiągnąć i zweryfikować…',
  'chat.goal.dialog.budgetLabel': 'Budżet tokenów',
+  'chat.goal.dialog.evaluationModelLabel': 'Model oceniający',
  'chat.goal.status.active': 'Aktywny',
  'chat.goal.status.evaluating': 'Ocenianie…',
  'chat.goal.status.paused': 'Wstrzymany',
@@ -1394,6 +1394,7 @@ export const dict: Record<I18nKey, string> = {
  "chat.goal.dialog.objectiveLabel": "Objetivo",
  "chat.goal.dialog.objectivePlaceholder": "Descreva o estado final que o agente deve alcançar e verificar…",
  "chat.goal.dialog.budgetLabel": "Orçamento de tokens",
+  "chat.goal.dialog.evaluationModelLabel": "Modelo de avaliação",
  "chat.goal.status.active": "Ativo",
  "chat.goal.status.evaluating": "Avaliando…",
  "chat.goal.status.paused": "Pausado",
@@ -1394,6 +1394,7 @@ export const dict: Record<I18nKey, string> = {
  "chat.goal.dialog.objectiveLabel": "Ціль",
  "chat.goal.dialog.objectivePlaceholder": "Опишіть кінцевий стан, якого агент має досягти та перевірити…",
  "chat.goal.dialog.budgetLabel": "Бюджет токенів",
+  "chat.goal.dialog.evaluationModelLabel": "Модель оцінювання",
  "chat.goal.status.active": "Активна",
  "chat.goal.status.evaluating": "Оцінювання…",
  "chat.goal.status.paused": "Призупинена",
@@ -1382,6 +1382,7 @@ export const dict: Record<I18nKey, string> = {
  'chat.goal.dialog.objectiveLabel': '目标',
  'chat.goal.dialog.objectivePlaceholder': '描述代理应达到并验证的最终状态…',
  'chat.goal.dialog.budgetLabel': '令牌预算',
+  'chat.goal.dialog.evaluationModelLabel': '评估模型',
  'chat.goal.status.active': '进行中',
  'chat.goal.status.evaluating': '评估中…',
  'chat.goal.status.paused': '已暂停',
@@ -1386,6 +1386,7 @@ export const dict: Record<I18nKey, string> = {
  'chat.goal.dialog.objectiveLabel': '目標',
  'chat.goal.dialog.objectivePlaceholder': '描述代理應達成並驗證的最終狀態…',
  'chat.goal.dialog.budgetLabel': '權杖預算',
+  'chat.goal.dialog.evaluationModelLabel': '評估模型',
  'chat.goal.status.active': '進行中',
  'chat.goal.status.evaluating': '評估中…',
  'chat.goal.status.paused': '已暫停',
@@ -23,6 +23,8 @@ export interface SessionGoalPayload {
  blockedStreak: number;
  note: string;
  statusReason: string;
+  evaluationProviderID: string;
+  evaluationModelID: string;
  lastAccountedMessageID: string;
  createdAt: number;
  updatedAt: number;
@@ -64,6 +66,8 @@ export function getSessionGoal(session: Session | null | undefined): SessionGoal
    blockedStreak: asCount(goal.blockedStreak),
    note: typeof goal.note === 'string' ? goal.note : '',
    statusReason: typeof goal.statusReason === 'string' ? goal.statusReason : '',
+    evaluationProviderID: typeof goal.evaluationProviderID === 'string' ? goal.evaluationProviderID : '',
+    evaluationModelID: typeof goal.evaluationModelID === 'string' ? goal.evaluationModelID : '',
    lastAccountedMessageID: typeof goal.lastAccountedMessageID === 'string' ? goal.lastAccountedMessageID : '',
    createdAt: typeof goal.createdAt === 'number' ? goal.createdAt : 0,
    updatedAt: typeof goal.updatedAt === 'number' ? goal.updatedAt : 0,
@@ -23,6 +23,8 @@ the web server and survives UI disconnects.
  auditFailStreak,         // consecutive failed/unavailable audit calls
  note,                    // latest audit progress note, <= 280 chars
  statusReason,            // why settled; 'resumed' is a kickoff signal from UI
+  evaluationProviderID,    // provider used by the latest successful audit
+  evaluationModelID,       // model used by the latest successful audit
  lastAccountedMessageID,  // incremental accounting cursor
  createdAt, updatedAt
 }
@@ -213,6 +213,8 @@ const parseGoalMetadata = (session) => {
    auditFailStreak: Number.isFinite(goal.auditFailStreak) && goal.auditFailStreak > 0 ? Math.floor(goal.auditFailStreak) : 0,
    note: typeof goal.note === 'string' ? goal.note.slice(0, NOTE_CHAR_LIMIT) : '',
    statusReason: typeof goal.statusReason === 'string' ? goal.statusReason.slice(0, REASON_CHAR_LIMIT) : '',
+    evaluationProviderID: typeof goal.evaluationProviderID === 'string' ? goal.evaluationProviderID : '',
+    evaluationModelID: typeof goal.evaluationModelID === 'string' ? goal.evaluationModelID : '',
    lastAccountedMessageID: typeof goal.lastAccountedMessageID === 'string' ? goal.lastAccountedMessageID : '',
    createdAt: Number.isFinite(goal.createdAt) ? goal.createdAt : 0,
    updatedAt: Number.isFinite(goal.updatedAt) ? goal.updatedAt : 0,
@@ -332,7 +334,7 @@ export const createSessionGoalRuntime = ({
    return nextGoal;
  };

-  const settleGoal = async ({ sessionId, directory, goal, status, statusReason, note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID }) => {
+  const settleGoal = async ({ sessionId, directory, goal, status, statusReason, note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID, evaluationProviderID, evaluationModelID }) => {
    const written = await writeGoal(sessionId, directory, goal.id, (current) => ({
      status,
      statusReason: clampText(statusReason, REASON_CHAR_LIMIT),
@@ -343,6 +345,8 @@ export const createSessionGoalRuntime = ({
      ...(tokensBaseline !== undefined ? { tokensBaseline } : {}),
      ...(tokensCommitted !== undefined ? { tokensCommitted } : {}),
      ...(lastAccountedMessageID ? { lastAccountedMessageID } : {}),
+      ...(evaluationProviderID ? { evaluationProviderID } : {}),
+      ...(evaluationModelID ? { evaluationModelID } : {}),
    }));
    if (!written) return;
    console.log(`[session-goal] ${sessionId} settled as ${status}${statusReason ? ` (${statusReason})` : ''}`);
@@ -378,13 +382,35 @@ export const createSessionGoalRuntime = ({
      });
      const structured = extractJsonObject(generated?.text);
      const verdict = typeof structured?.verdict === 'string' ? structured.verdict.trim().toLowerCase() : '';
-      if (!['continue', 'complete', 'blocked'].includes(verdict)) return null;
+      if (!structured || !['continue', 'complete', 'blocked'].includes(verdict)) {
+        console.warn('[session-goal:diagnostic] audit parse failed', {
+          sessionId: lastAssistantInfo?.sessionID ?? null,
+          provider: generated?.providerID ?? null,
+          model: generated?.modelID ?? null,
+          outputChars: typeof generated?.text === 'string' ? generated.text.length : 0,
+          jsonObjectFound: Boolean(structured),
+          verdict: verdict || null,
+        });
+        return null;
+      }
+      console.log('[session-goal:diagnostic] audit verdict', {
+        sessionId: lastAssistantInfo?.sessionID ?? null,
+        provider: generated?.providerID ?? null,
+        model: generated?.modelID ?? null,
+        outputChars: generated.text.length,
+        verdict,
+      });
      let note = clampText(structured?.note, NOTE_CHAR_LIMIT);
      if (note && hasScriptMismatch(note, `${goal.objective}\n${assistantText}`)) {
        console.warn('[session-goal] dropped audit note: language mismatch with objective');
        note = '';
      }
-      return { verdict, note };
+      return {
+        verdict,
+        note,
+        evaluationProviderID: generated.providerID,
+        evaluationModelID: generated.modelID,
+      };
    } catch (error) {
      // No authenticated small model (404) or a transient failure — the loop
      // still terminates via markers, budget, and the turn cap.
@@ -656,15 +682,22 @@ export const createSessionGoalRuntime = ({
      if (audit?.verdict === 'complete') {
        await settleGoal({
          sessionId, directory, goal, status: 'complete', statusReason: 'verified by audit', note: audit.note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID,
+          evaluationProviderID: audit.evaluationProviderID, evaluationModelID: audit.evaluationModelID,
        });
        return;
      }

      if (audit?.verdict === 'blocked') {
        blockedStreak = goal.blockedStreak + 1;
+        console.warn('[session-goal:diagnostic] blocked audit streak', {
+          sessionId,
+          blockedStreak,
+          blockedStreakLimit: BLOCKED_STREAK_LIMIT,
+        });
        if (blockedStreak >= BLOCKED_STREAK_LIMIT) {
          await settleGoal({
            sessionId, directory, goal, status: 'blocked', statusReason: audit.note || 'blocked per audit', note: audit.note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID,
+            evaluationProviderID: audit.evaluationProviderID, evaluationModelID: audit.evaluationModelID,
          });
          return;
        }
@@ -684,6 +717,8 @@ export const createSessionGoalRuntime = ({
      auditFailStreak,
      statusReason: '',
      ...(audit?.note ? { note: audit.note } : {}),
+      ...(audit?.evaluationProviderID ? { evaluationProviderID: audit.evaluationProviderID } : {}),
+      ...(audit?.evaluationModelID ? { evaluationModelID: audit.evaluationModelID } : {}),
    }));
    if (!written) {
      console.log('[session-goal] goal changed during tick, dropping continuation');
@@ -123,7 +123,7 @@ describe('session goal live activity gate', () => {
    const requests = [];
    const fetchImpl = vi.fn(async (input, init = {}) => {
      const pathname = requestPath(input);
-      requests.push({ pathname, method: init.method ?? 'GET' });
+      requests.push({ pathname, method: init.method ?? 'GET', body: init.body });
      if (pathname === `/session/${SESSION_ID}` && init.method === 'PATCH') return jsonResponse(session);
      if (pathname === `/session/${SESSION_ID}`) return jsonResponse(session);
      if (pathname === '/session/status') return jsonResponse({});
@@ -147,6 +147,8 @@ describe('session goal live activity gate', () => {
    const service = {
      generateSmallModelText: vi.fn(async () => ({
        text: '{"verdict":"complete","note":"Task verified complete"}',
+        providerID: 'provider',
+        modelID: 'model',
      })),
    };
    vi.stubGlobal('fetch', fetchImpl);
@@ -164,7 +166,14 @@ describe('session goal live activity gate', () => {
    await vi.advanceTimersByTimeAsync(10);

    expect(service.generateSmallModelText).toHaveBeenCalledOnce();
-    expect(requests).toContainEqual({ pathname: `/session/${SESSION_ID}`, method: 'PATCH' });
+    const patch = requests.find((request) => request.pathname === `/session/${SESSION_ID}` && request.method === 'PATCH');
+    expect(patch).toBeDefined();
+    const writtenGoal = JSON.parse(patch.body).metadata.openchamber.goal;
+    expect(writtenGoal).toMatchObject({
+      status: 'complete',
+      evaluationProviderID: 'provider',
+      evaluationModelID: 'model',
+    });
    runtime.stop();
  });
 });
@@ -49,7 +49,12 @@ other runtime API.
  - Everything else: OpenAI-compatible `/chat/completions` against the
    provider's base URL, resolved from (1) `provider.<id>.options.baseURL`
    in the OpenCode config, (2) the hardcoded `https://api.openai.com/v1`
-    endpoint, or (3) the provider's `api` field from the models.dev catalog.
+     endpoint, or (3) the provider's `api` field from the models.dev catalog.
+  - `[small-model:diagnostic]` logs record provider/model, input character
+    counts, output budget, thinking toggle, HTTP/finish status, and
+    content/reasoning lengths without logging prompts, response text, or
+    credentials. Goal audit parsing similarly emits
+    `[session-goal:diagnostic]` structural verdict metadata.
 - `catalog.js` — models.dev catalog via the shared in-process cache
  (`../opencode/models-metadata.js`, also serving
  `/api/openchamber/models-metadata`).
@@ -104,6 +104,15 @@ const ensureFreshOpenaiOauth = async (entry) => {

 const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system, maxOutputTokens, providerLabel, extraBody }) => {
  const trimmedBase = baseURL.replace(/\/+$/, '');
+  console.log('[small-model:diagnostic] request', {
+    provider: providerLabel,
+    model: modelID,
+    maxOutputTokens,
+    thinkingDisabled: extraBody?.thinking?.type === 'disabled',
+    promptChars: prompt.length,
+    systemChars: system?.length ?? 0,
+    inputChars: prompt.length + (system?.length ?? 0),
+  });
  const response = await fetch(`${trimmedBase}/chat/completions`, {
    method: 'POST',
    headers: {
@@ -123,11 +132,29 @@ const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system,
    }),
    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
  });
+  console.log('[small-model:diagnostic] response', {
+    provider: providerLabel,
+    model: modelID,
+    httpStatus: response.status,
+    ok: response.ok,
+  });
  if (!response.ok) {
    throw await httpError(response, providerLabel);
  }
  const payload = await response.json();
  const message = payload?.choices?.[0]?.message;
+  console.log('[small-model:diagnostic] completion', {
+    provider: providerLabel,
+    model: modelID,
+    finishReason: payload?.choices?.[0]?.finish_reason ?? null,
+    contentType: Array.isArray(message?.content) ? 'parts' : typeof message?.content,
+    contentChars: typeof message?.content === 'string'
+      ? message.content.length
+      : Array.isArray(message?.content)
+        ? message.content.reduce((total, part) => total + (typeof part?.text === 'string' ? part.text.length : 0), 0)
+        : 0,
+    reasoningChars: typeof message?.reasoning_content === 'string' ? message.reasoning_content.length : 0,
+  });

  // Providers disagree on the content shape: plain string, an array of
  // typed parts, or (thinking models) an empty content with the budget spent