diff --git a/packages/ui/src/components/chat/SessionGoalDialog.tsx b/packages/ui/src/components/chat/SessionGoalDialog.tsx index 5192db36..2ed190c7 100644 --- a/packages/ui/src/components/chat/SessionGoalDialog.tsx +++ b/packages/ui/src/components/chat/SessionGoalDialog.tsx @@ -117,6 +117,14 @@ export function SessionGoalDialog({ open, onOpenChange, sessionId, directory }: {goal.statusReason && (goal.status === 'blocked' || goal.status === 'budgetLimited') ? (

{goal.statusReason}

) : null} + {goal.evaluationProviderID || goal.evaluationModelID ? ( +
+ {t('chat.goal.dialog.evaluationModelLabel')} + + {[goal.evaluationProviderID, goal.evaluationModelID].filter(Boolean).join('/')} + +
+ ) : null} )} diff --git a/packages/ui/src/lib/i18n/messages/en.ts b/packages/ui/src/lib/i18n/messages/en.ts index cd8f10d6..12e193ca 100644 --- a/packages/ui/src/lib/i18n/messages/en.ts +++ b/packages/ui/src/lib/i18n/messages/en.ts @@ -1416,6 +1416,7 @@ export const dict = { 'chat.goal.dialog.objectiveLabel': 'Objective', 'chat.goal.dialog.objectivePlaceholder': 'Describe the end state the agent should reach and verify…', 'chat.goal.dialog.budgetLabel': 'Token budget', + 'chat.goal.dialog.evaluationModelLabel': 'Evaluation model', 'chat.goal.status.active': 'Active', 'chat.goal.status.evaluating': 'Evaluating…', 'chat.goal.status.paused': 'Paused', diff --git a/packages/ui/src/lib/i18n/messages/es.ts b/packages/ui/src/lib/i18n/messages/es.ts index 00c4d74c..0f1f84dd 100644 --- a/packages/ui/src/lib/i18n/messages/es.ts +++ b/packages/ui/src/lib/i18n/messages/es.ts @@ -1394,6 +1394,7 @@ export const dict: Record = { "chat.goal.dialog.objectiveLabel": "Objetivo", "chat.goal.dialog.objectivePlaceholder": "Describe el estado final que el agente debe alcanzar y verificar…", "chat.goal.dialog.budgetLabel": "Presupuesto de tokens", + "chat.goal.dialog.evaluationModelLabel": "Modelo de evaluación", "chat.goal.status.active": "Activo", "chat.goal.status.evaluating": "Evaluando…", "chat.goal.status.paused": "En pausa", diff --git a/packages/ui/src/lib/i18n/messages/fr.ts b/packages/ui/src/lib/i18n/messages/fr.ts index 1fbd1da7..f82a4063 100644 --- a/packages/ui/src/lib/i18n/messages/fr.ts +++ b/packages/ui/src/lib/i18n/messages/fr.ts @@ -1236,6 +1236,7 @@ export const dict = { 'chat.goal.dialog.objectiveLabel': 'Objectif', 'chat.goal.dialog.objectivePlaceholder': 'Décrivez l\'état final que l\'agent doit atteindre et vérifier…', 'chat.goal.dialog.budgetLabel': 'Budget de tokens', + 'chat.goal.dialog.evaluationModelLabel': 'Modèle d’évaluation', 'chat.goal.status.active': 'Actif', 'chat.goal.status.evaluating': 'Évaluation…', 'chat.goal.status.paused': 'En pause', diff --git a/packages/ui/src/lib/i18n/messages/ja.ts b/packages/ui/src/lib/i18n/messages/ja.ts index 98c7addf..a105dd50 100644 --- a/packages/ui/src/lib/i18n/messages/ja.ts +++ b/packages/ui/src/lib/i18n/messages/ja.ts @@ -1412,6 +1412,7 @@ export const dict: Record = { 'chat.goal.dialog.objectiveLabel': '目標', 'chat.goal.dialog.objectivePlaceholder': 'エージェントが到達して検証すべき最終状態を記述してください…', 'chat.goal.dialog.budgetLabel': 'トークン予算', + 'chat.goal.dialog.evaluationModelLabel': '評価モデル', 'chat.goal.status.active': '進行中', 'chat.goal.status.evaluating': '評価中…', 'chat.goal.status.paused': '一時停止', diff --git a/packages/ui/src/lib/i18n/messages/ko.ts b/packages/ui/src/lib/i18n/messages/ko.ts index 9fd2883c..0ab296d4 100644 --- a/packages/ui/src/lib/i18n/messages/ko.ts +++ b/packages/ui/src/lib/i18n/messages/ko.ts @@ -1418,6 +1418,7 @@ export const dict: Record = { 'chat.goal.dialog.objectiveLabel': '목표', 'chat.goal.dialog.objectivePlaceholder': '에이전트가 도달하고 검증해야 할 최종 상태를 설명하세요…', 'chat.goal.dialog.budgetLabel': '토큰 예산', + 'chat.goal.dialog.evaluationModelLabel': '평가 모델', 'chat.goal.status.active': '진행 중', 'chat.goal.status.evaluating': '평가 중…', 'chat.goal.status.paused': '일시 중지됨', diff --git a/packages/ui/src/lib/i18n/messages/pl.ts b/packages/ui/src/lib/i18n/messages/pl.ts index ce5a3be9..d2d03be1 100644 --- a/packages/ui/src/lib/i18n/messages/pl.ts +++ b/packages/ui/src/lib/i18n/messages/pl.ts @@ -2104,6 +2104,7 @@ export const dict: Record = { 'chat.goal.dialog.objectiveLabel': 'Cel', 'chat.goal.dialog.objectivePlaceholder': 'Opisz stan końcowy, który agent ma osiągnąć i zweryfikować…', 'chat.goal.dialog.budgetLabel': 'Budżet tokenów', + 'chat.goal.dialog.evaluationModelLabel': 'Model oceniający', 'chat.goal.status.active': 'Aktywny', 'chat.goal.status.evaluating': 'Ocenianie…', 'chat.goal.status.paused': 'Wstrzymany', diff --git a/packages/ui/src/lib/i18n/messages/pt-BR.ts b/packages/ui/src/lib/i18n/messages/pt-BR.ts index c0554bdf..5343dc02 100644 --- a/packages/ui/src/lib/i18n/messages/pt-BR.ts +++ b/packages/ui/src/lib/i18n/messages/pt-BR.ts @@ -1394,6 +1394,7 @@ export const dict: Record = { "chat.goal.dialog.objectiveLabel": "Objetivo", "chat.goal.dialog.objectivePlaceholder": "Descreva o estado final que o agente deve alcançar e verificar…", "chat.goal.dialog.budgetLabel": "Orçamento de tokens", + "chat.goal.dialog.evaluationModelLabel": "Modelo de avaliação", "chat.goal.status.active": "Ativo", "chat.goal.status.evaluating": "Avaliando…", "chat.goal.status.paused": "Pausado", diff --git a/packages/ui/src/lib/i18n/messages/uk.ts b/packages/ui/src/lib/i18n/messages/uk.ts index 6d9f9432..d044c2d0 100644 --- a/packages/ui/src/lib/i18n/messages/uk.ts +++ b/packages/ui/src/lib/i18n/messages/uk.ts @@ -1394,6 +1394,7 @@ export const dict: Record = { "chat.goal.dialog.objectiveLabel": "Ціль", "chat.goal.dialog.objectivePlaceholder": "Опишіть кінцевий стан, якого агент має досягти та перевірити…", "chat.goal.dialog.budgetLabel": "Бюджет токенів", + "chat.goal.dialog.evaluationModelLabel": "Модель оцінювання", "chat.goal.status.active": "Активна", "chat.goal.status.evaluating": "Оцінювання…", "chat.goal.status.paused": "Призупинена", diff --git a/packages/ui/src/lib/i18n/messages/zh-CN.ts b/packages/ui/src/lib/i18n/messages/zh-CN.ts index f13a037c..49fe02ea 100644 --- a/packages/ui/src/lib/i18n/messages/zh-CN.ts +++ b/packages/ui/src/lib/i18n/messages/zh-CN.ts @@ -1382,6 +1382,7 @@ export const dict: Record = { 'chat.goal.dialog.objectiveLabel': '目标', 'chat.goal.dialog.objectivePlaceholder': '描述代理应达到并验证的最终状态…', 'chat.goal.dialog.budgetLabel': '令牌预算', + 'chat.goal.dialog.evaluationModelLabel': '评估模型', 'chat.goal.status.active': '进行中', 'chat.goal.status.evaluating': '评估中…', 'chat.goal.status.paused': '已暂停', diff --git a/packages/ui/src/lib/i18n/messages/zh-TW.ts b/packages/ui/src/lib/i18n/messages/zh-TW.ts index 1451a6c8..a169b976 100644 --- a/packages/ui/src/lib/i18n/messages/zh-TW.ts +++ b/packages/ui/src/lib/i18n/messages/zh-TW.ts @@ -1386,6 +1386,7 @@ export const dict: Record = { 'chat.goal.dialog.objectiveLabel': '目標', 'chat.goal.dialog.objectivePlaceholder': '描述代理應達成並驗證的最終狀態…', 'chat.goal.dialog.budgetLabel': '權杖預算', + 'chat.goal.dialog.evaluationModelLabel': '評估模型', 'chat.goal.status.active': '進行中', 'chat.goal.status.evaluating': '評估中…', 'chat.goal.status.paused': '已暫停', diff --git a/packages/ui/src/lib/sessionGoalMetadata.ts b/packages/ui/src/lib/sessionGoalMetadata.ts index a372b8c7..1b92f368 100644 --- a/packages/ui/src/lib/sessionGoalMetadata.ts +++ b/packages/ui/src/lib/sessionGoalMetadata.ts @@ -23,6 +23,8 @@ export interface SessionGoalPayload { blockedStreak: number; note: string; statusReason: string; + evaluationProviderID: string; + evaluationModelID: string; lastAccountedMessageID: string; createdAt: number; updatedAt: number; @@ -64,6 +66,8 @@ export function getSessionGoal(session: Session | null | undefined): SessionGoal blockedStreak: asCount(goal.blockedStreak), note: typeof goal.note === 'string' ? goal.note : '', statusReason: typeof goal.statusReason === 'string' ? goal.statusReason : '', + evaluationProviderID: typeof goal.evaluationProviderID === 'string' ? goal.evaluationProviderID : '', + evaluationModelID: typeof goal.evaluationModelID === 'string' ? goal.evaluationModelID : '', lastAccountedMessageID: typeof goal.lastAccountedMessageID === 'string' ? goal.lastAccountedMessageID : '', createdAt: typeof goal.createdAt === 'number' ? goal.createdAt : 0, updatedAt: typeof goal.updatedAt === 'number' ? goal.updatedAt : 0, diff --git a/packages/web/server/lib/session-goal/DOCUMENTATION.md b/packages/web/server/lib/session-goal/DOCUMENTATION.md index 32ecb786..9709b492 100644 --- a/packages/web/server/lib/session-goal/DOCUMENTATION.md +++ b/packages/web/server/lib/session-goal/DOCUMENTATION.md @@ -23,6 +23,8 @@ the web server and survives UI disconnects. auditFailStreak, // consecutive failed/unavailable audit calls note, // latest audit progress note, <= 280 chars statusReason, // why settled; 'resumed' is a kickoff signal from UI + evaluationProviderID, // provider used by the latest successful audit + evaluationModelID, // model used by the latest successful audit lastAccountedMessageID, // incremental accounting cursor createdAt, updatedAt } diff --git a/packages/web/server/lib/session-goal/runtime.js b/packages/web/server/lib/session-goal/runtime.js index 98c36cd6..e6916997 100644 --- a/packages/web/server/lib/session-goal/runtime.js +++ b/packages/web/server/lib/session-goal/runtime.js @@ -213,6 +213,8 @@ const parseGoalMetadata = (session) => { auditFailStreak: Number.isFinite(goal.auditFailStreak) && goal.auditFailStreak > 0 ? Math.floor(goal.auditFailStreak) : 0, note: typeof goal.note === 'string' ? goal.note.slice(0, NOTE_CHAR_LIMIT) : '', statusReason: typeof goal.statusReason === 'string' ? goal.statusReason.slice(0, REASON_CHAR_LIMIT) : '', + evaluationProviderID: typeof goal.evaluationProviderID === 'string' ? goal.evaluationProviderID : '', + evaluationModelID: typeof goal.evaluationModelID === 'string' ? goal.evaluationModelID : '', lastAccountedMessageID: typeof goal.lastAccountedMessageID === 'string' ? goal.lastAccountedMessageID : '', createdAt: Number.isFinite(goal.createdAt) ? goal.createdAt : 0, updatedAt: Number.isFinite(goal.updatedAt) ? goal.updatedAt : 0, @@ -332,7 +334,7 @@ export const createSessionGoalRuntime = ({ return nextGoal; }; - const settleGoal = async ({ sessionId, directory, goal, status, statusReason, note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID }) => { + const settleGoal = async ({ sessionId, directory, goal, status, statusReason, note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID, evaluationProviderID, evaluationModelID }) => { const written = await writeGoal(sessionId, directory, goal.id, (current) => ({ status, statusReason: clampText(statusReason, REASON_CHAR_LIMIT), @@ -343,6 +345,8 @@ export const createSessionGoalRuntime = ({ ...(tokensBaseline !== undefined ? { tokensBaseline } : {}), ...(tokensCommitted !== undefined ? { tokensCommitted } : {}), ...(lastAccountedMessageID ? { lastAccountedMessageID } : {}), + ...(evaluationProviderID ? { evaluationProviderID } : {}), + ...(evaluationModelID ? { evaluationModelID } : {}), })); if (!written) return; console.log(`[session-goal] ${sessionId} settled as ${status}${statusReason ? ` (${statusReason})` : ''}`); @@ -378,13 +382,35 @@ export const createSessionGoalRuntime = ({ }); const structured = extractJsonObject(generated?.text); const verdict = typeof structured?.verdict === 'string' ? structured.verdict.trim().toLowerCase() : ''; - if (!['continue', 'complete', 'blocked'].includes(verdict)) return null; + if (!structured || !['continue', 'complete', 'blocked'].includes(verdict)) { + console.warn('[session-goal:diagnostic] audit parse failed', { + sessionId: lastAssistantInfo?.sessionID ?? null, + provider: generated?.providerID ?? null, + model: generated?.modelID ?? null, + outputChars: typeof generated?.text === 'string' ? generated.text.length : 0, + jsonObjectFound: Boolean(structured), + verdict: verdict || null, + }); + return null; + } + console.log('[session-goal:diagnostic] audit verdict', { + sessionId: lastAssistantInfo?.sessionID ?? null, + provider: generated?.providerID ?? null, + model: generated?.modelID ?? null, + outputChars: generated.text.length, + verdict, + }); let note = clampText(structured?.note, NOTE_CHAR_LIMIT); if (note && hasScriptMismatch(note, `${goal.objective}\n${assistantText}`)) { console.warn('[session-goal] dropped audit note: language mismatch with objective'); note = ''; } - return { verdict, note }; + return { + verdict, + note, + evaluationProviderID: generated.providerID, + evaluationModelID: generated.modelID, + }; } catch (error) { // No authenticated small model (404) or a transient failure — the loop // still terminates via markers, budget, and the turn cap. @@ -656,15 +682,22 @@ export const createSessionGoalRuntime = ({ if (audit?.verdict === 'complete') { await settleGoal({ sessionId, directory, goal, status: 'complete', statusReason: 'verified by audit', note: audit.note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID, + evaluationProviderID: audit.evaluationProviderID, evaluationModelID: audit.evaluationModelID, }); return; } if (audit?.verdict === 'blocked') { blockedStreak = goal.blockedStreak + 1; + console.warn('[session-goal:diagnostic] blocked audit streak', { + sessionId, + blockedStreak, + blockedStreakLimit: BLOCKED_STREAK_LIMIT, + }); if (blockedStreak >= BLOCKED_STREAK_LIMIT) { await settleGoal({ sessionId, directory, goal, status: 'blocked', statusReason: audit.note || 'blocked per audit', note: audit.note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID, + evaluationProviderID: audit.evaluationProviderID, evaluationModelID: audit.evaluationModelID, }); return; } @@ -684,6 +717,8 @@ export const createSessionGoalRuntime = ({ auditFailStreak, statusReason: '', ...(audit?.note ? { note: audit.note } : {}), + ...(audit?.evaluationProviderID ? { evaluationProviderID: audit.evaluationProviderID } : {}), + ...(audit?.evaluationModelID ? { evaluationModelID: audit.evaluationModelID } : {}), })); if (!written) { console.log('[session-goal] goal changed during tick, dropping continuation'); diff --git a/packages/web/server/lib/session-goal/runtime.test.js b/packages/web/server/lib/session-goal/runtime.test.js index 40f6a29f..e091c8c4 100644 --- a/packages/web/server/lib/session-goal/runtime.test.js +++ b/packages/web/server/lib/session-goal/runtime.test.js @@ -123,7 +123,7 @@ describe('session goal live activity gate', () => { const requests = []; const fetchImpl = vi.fn(async (input, init = {}) => { const pathname = requestPath(input); - requests.push({ pathname, method: init.method ?? 'GET' }); + requests.push({ pathname, method: init.method ?? 'GET', body: init.body }); if (pathname === `/session/${SESSION_ID}` && init.method === 'PATCH') return jsonResponse(session); if (pathname === `/session/${SESSION_ID}`) return jsonResponse(session); if (pathname === '/session/status') return jsonResponse({}); @@ -147,6 +147,8 @@ describe('session goal live activity gate', () => { const service = { generateSmallModelText: vi.fn(async () => ({ text: '{"verdict":"complete","note":"Task verified complete"}', + providerID: 'provider', + modelID: 'model', })), }; vi.stubGlobal('fetch', fetchImpl); @@ -164,7 +166,14 @@ describe('session goal live activity gate', () => { await vi.advanceTimersByTimeAsync(10); expect(service.generateSmallModelText).toHaveBeenCalledOnce(); - expect(requests).toContainEqual({ pathname: `/session/${SESSION_ID}`, method: 'PATCH' }); + const patch = requests.find((request) => request.pathname === `/session/${SESSION_ID}` && request.method === 'PATCH'); + expect(patch).toBeDefined(); + const writtenGoal = JSON.parse(patch.body).metadata.openchamber.goal; + expect(writtenGoal).toMatchObject({ + status: 'complete', + evaluationProviderID: 'provider', + evaluationModelID: 'model', + }); runtime.stop(); }); }); diff --git a/packages/web/server/lib/small-model/DOCUMENTATION.md b/packages/web/server/lib/small-model/DOCUMENTATION.md index fc2a1ba7..7c103a13 100644 --- a/packages/web/server/lib/small-model/DOCUMENTATION.md +++ b/packages/web/server/lib/small-model/DOCUMENTATION.md @@ -49,7 +49,12 @@ other runtime API. - Everything else: OpenAI-compatible `/chat/completions` against the provider's base URL, resolved from (1) `provider..options.baseURL` in the OpenCode config, (2) the hardcoded `https://api.openai.com/v1` - endpoint, or (3) the provider's `api` field from the models.dev catalog. + endpoint, or (3) the provider's `api` field from the models.dev catalog. + - `[small-model:diagnostic]` logs record provider/model, input character + counts, output budget, thinking toggle, HTTP/finish status, and + content/reasoning lengths without logging prompts, response text, or + credentials. Goal audit parsing similarly emits + `[session-goal:diagnostic]` structural verdict metadata. - `catalog.js` — models.dev catalog via the shared in-process cache (`../opencode/models-metadata.js`, also serving `/api/openchamber/models-metadata`). diff --git a/packages/web/server/lib/small-model/call.js b/packages/web/server/lib/small-model/call.js index ff38a1ca..858fba9b 100644 --- a/packages/web/server/lib/small-model/call.js +++ b/packages/web/server/lib/small-model/call.js @@ -104,6 +104,15 @@ const ensureFreshOpenaiOauth = async (entry) => { const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system, maxOutputTokens, providerLabel, extraBody }) => { const trimmedBase = baseURL.replace(/\/+$/, ''); + console.log('[small-model:diagnostic] request', { + provider: providerLabel, + model: modelID, + maxOutputTokens, + thinkingDisabled: extraBody?.thinking?.type === 'disabled', + promptChars: prompt.length, + systemChars: system?.length ?? 0, + inputChars: prompt.length + (system?.length ?? 0), + }); const response = await fetch(`${trimmedBase}/chat/completions`, { method: 'POST', headers: { @@ -123,11 +132,29 @@ const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system, }), signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), }); + console.log('[small-model:diagnostic] response', { + provider: providerLabel, + model: modelID, + httpStatus: response.status, + ok: response.ok, + }); if (!response.ok) { throw await httpError(response, providerLabel); } const payload = await response.json(); const message = payload?.choices?.[0]?.message; + console.log('[small-model:diagnostic] completion', { + provider: providerLabel, + model: modelID, + finishReason: payload?.choices?.[0]?.finish_reason ?? null, + contentType: Array.isArray(message?.content) ? 'parts' : typeof message?.content, + contentChars: typeof message?.content === 'string' + ? message.content.length + : Array.isArray(message?.content) + ? message.content.reduce((total, part) => total + (typeof part?.text === 'string' ? part.text.length : 0), 0) + : 0, + reasoningChars: typeof message?.reasoning_content === 'string' ? message.reasoning_content.length : 0, + }); // Providers disagree on the content shape: plain string, an array of // typed parts, or (thinking models) an empty content with the budget spent