feat: surface session goal evaluation model and add diagnostics

Shows the evaluation provider/model used for the latest successful goal audit in the UI.
Persists evaluation model metadata on session goals and covers it in tests.
Adds diagnostic logging for small-model calls and session-goal audit parsing.
This commit is contained in:
Bohdan Triapitsyn
2026-07-15 08:56:10 +03:00
parent 4eac90ad44
commit e48a9397f1
17 changed files with 106 additions and 6 deletions
@@ -49,7 +49,12 @@ other runtime API.
- Everything else: OpenAI-compatible `/chat/completions` against the
provider's base URL, resolved from (1) `provider.<id>.options.baseURL`
in the OpenCode config, (2) the hardcoded `https://api.openai.com/v1`
endpoint, or (3) the provider's `api` field from the models.dev catalog.
endpoint, or (3) the provider's `api` field from the models.dev catalog.
- `[small-model:diagnostic]` logs record provider/model, input character
counts, output budget, thinking toggle, HTTP/finish status, and
content/reasoning lengths without logging prompts, response text, or
credentials. Goal audit parsing similarly emits
`[session-goal:diagnostic]` structural verdict metadata.
- `catalog.js` — models.dev catalog via the shared in-process cache
(`../opencode/models-metadata.js`, also serving
`/api/openchamber/models-metadata`).
@@ -104,6 +104,15 @@ const ensureFreshOpenaiOauth = async (entry) => {
const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system, maxOutputTokens, providerLabel, extraBody }) => {
const trimmedBase = baseURL.replace(/\/+$/, '');
console.log('[small-model:diagnostic] request', {
provider: providerLabel,
model: modelID,
maxOutputTokens,
thinkingDisabled: extraBody?.thinking?.type === 'disabled',
promptChars: prompt.length,
systemChars: system?.length ?? 0,
inputChars: prompt.length + (system?.length ?? 0),
});
const response = await fetch(`${trimmedBase}/chat/completions`, {
method: 'POST',
headers: {
@@ -123,11 +132,29 @@ const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system,
}),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
console.log('[small-model:diagnostic] response', {
provider: providerLabel,
model: modelID,
httpStatus: response.status,
ok: response.ok,
});
if (!response.ok) {
throw await httpError(response, providerLabel);
}
const payload = await response.json();
const message = payload?.choices?.[0]?.message;
console.log('[small-model:diagnostic] completion', {
provider: providerLabel,
model: modelID,
finishReason: payload?.choices?.[0]?.finish_reason ?? null,
contentType: Array.isArray(message?.content) ? 'parts' : typeof message?.content,
contentChars: typeof message?.content === 'string'
? message.content.length
: Array.isArray(message?.content)
? message.content.reduce((total, part) => total + (typeof part?.text === 'string' ? part.text.length : 0), 0)
: 0,
reasoningChars: typeof message?.reasoning_content === 'string' ? message.reasoning_content.length : 0,
});
// Providers disagree on the content shape: plain string, an array of
// typed parts, or (thinking models) an empty content with the budget spent