feat: surface session goal evaluation model and add diagnostics
Shows the evaluation provider/model used for the latest successful goal audit in the UI. Persists evaluation model metadata on session goals and covers it in tests. Adds diagnostic logging for small-model calls and session-goal audit parsing.
This commit is contained in:
@@ -49,7 +49,12 @@ other runtime API.
|
||||
- Everything else: OpenAI-compatible `/chat/completions` against the
|
||||
provider's base URL, resolved from (1) `provider.<id>.options.baseURL`
|
||||
in the OpenCode config, (2) the hardcoded `https://api.openai.com/v1`
|
||||
endpoint, or (3) the provider's `api` field from the models.dev catalog.
|
||||
endpoint, or (3) the provider's `api` field from the models.dev catalog.
|
||||
- `[small-model:diagnostic]` logs record provider/model, input character
|
||||
counts, output budget, thinking toggle, HTTP/finish status, and
|
||||
content/reasoning lengths without logging prompts, response text, or
|
||||
credentials. Goal audit parsing similarly emits
|
||||
`[session-goal:diagnostic]` structural verdict metadata.
|
||||
- `catalog.js` — models.dev catalog via the shared in-process cache
|
||||
(`../opencode/models-metadata.js`, also serving
|
||||
`/api/openchamber/models-metadata`).
|
||||
|
||||
@@ -104,6 +104,15 @@ const ensureFreshOpenaiOauth = async (entry) => {
|
||||
|
||||
const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system, maxOutputTokens, providerLabel, extraBody }) => {
|
||||
const trimmedBase = baseURL.replace(/\/+$/, '');
|
||||
console.log('[small-model:diagnostic] request', {
|
||||
provider: providerLabel,
|
||||
model: modelID,
|
||||
maxOutputTokens,
|
||||
thinkingDisabled: extraBody?.thinking?.type === 'disabled',
|
||||
promptChars: prompt.length,
|
||||
systemChars: system?.length ?? 0,
|
||||
inputChars: prompt.length + (system?.length ?? 0),
|
||||
});
|
||||
const response = await fetch(`${trimmedBase}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
@@ -123,11 +132,29 @@ const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system,
|
||||
}),
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
||||
});
|
||||
console.log('[small-model:diagnostic] response', {
|
||||
provider: providerLabel,
|
||||
model: modelID,
|
||||
httpStatus: response.status,
|
||||
ok: response.ok,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw await httpError(response, providerLabel);
|
||||
}
|
||||
const payload = await response.json();
|
||||
const message = payload?.choices?.[0]?.message;
|
||||
console.log('[small-model:diagnostic] completion', {
|
||||
provider: providerLabel,
|
||||
model: modelID,
|
||||
finishReason: payload?.choices?.[0]?.finish_reason ?? null,
|
||||
contentType: Array.isArray(message?.content) ? 'parts' : typeof message?.content,
|
||||
contentChars: typeof message?.content === 'string'
|
||||
? message.content.length
|
||||
: Array.isArray(message?.content)
|
||||
? message.content.reduce((total, part) => total + (typeof part?.text === 'string' ? part.text.length : 0), 0)
|
||||
: 0,
|
||||
reasoningChars: typeof message?.reasoning_content === 'string' ? message.reasoning_content.length : 0,
|
||||
});
|
||||
|
||||
// Providers disagree on the content shape: plain string, an array of
|
||||
// typed parts, or (thinking models) an empty content with the budget spent
|
||||
|
||||
Reference in New Issue
Block a user