BuzzbeeSCD/openchamber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: surface session goal evaluation model and add diagnostics

Browse Source 
Shows the evaluation provider/model used for the latest successful goal audit in the UI.
Persists evaluation model metadata on session goals and covers it in tests.
Adds diagnostic logging for small-model calls and session-goal audit parsing.
This commit is contained in:
Bohdan Triapitsyn
2026-07-15 08:56:10 +03:00
parent 4eac90ad44
commit e48a9397f1
17 changed files with 106 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/server/lib/session-goal/DOCUMENTATION.md
+2
View File
@@ -23,6 +23,8 @@ the web server and survives UI disconnects.
auditFailStreak, // consecutive failed/unavailable audit calls
note, // latest audit progress note, <= 280 chars
statusReason, // why settled; 'resumed' is a kickoff signal from UI
evaluationProviderID, // provider used by the latest successful audit
evaluationModelID, // model used by the latest successful audit
lastAccountedMessageID, // incremental accounting cursor
createdAt, updatedAt
}
packages/web/server/lib/session-goal/runtime.js
+38 -3
View File
@@ -213,6 +213,8 @@ const parseGoalMetadata = (session) => {
auditFailStreak: Number.isFinite(goal.auditFailStreak) && goal.auditFailStreak > 0 ? Math.floor(goal.auditFailStreak) : 0,
note: typeof goal.note === 'string' ? goal.note.slice(0, NOTE_CHAR_LIMIT) : '',
statusReason: typeof goal.statusReason === 'string' ? goal.statusReason.slice(0, REASON_CHAR_LIMIT) : '',
evaluationProviderID: typeof goal.evaluationProviderID === 'string' ? goal.evaluationProviderID : '',
evaluationModelID: typeof goal.evaluationModelID === 'string' ? goal.evaluationModelID : '',
lastAccountedMessageID: typeof goal.lastAccountedMessageID === 'string' ? goal.lastAccountedMessageID : '',
createdAt: Number.isFinite(goal.createdAt) ? goal.createdAt : 0,
updatedAt: Number.isFinite(goal.updatedAt) ? goal.updatedAt : 0,
@@ -332,7 +334,7 @@ export const createSessionGoalRuntime = ({
return nextGoal;
};
const settleGoal = async ({ sessionId, directory, goal, status, statusReason, note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID }) => {
const settleGoal = async ({ sessionId, directory, goal, status, statusReason, note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID, evaluationProviderID, evaluationModelID }) => {
const written = await writeGoal(sessionId, directory, goal.id, (current) => ({
status,
statusReason: clampText(statusReason, REASON_CHAR_LIMIT),
@@ -343,6 +345,8 @@ export const createSessionGoalRuntime = ({
...(tokensBaseline !== undefined ? { tokensBaseline } : {}),
...(tokensCommitted !== undefined ? { tokensCommitted } : {}),
...(lastAccountedMessageID ? { lastAccountedMessageID } : {}),
...(evaluationProviderID ? { evaluationProviderID } : {}),
...(evaluationModelID ? { evaluationModelID } : {}),
}));
if (!written) return;
console.log(`[session-goal] ${sessionId} settled as ${status}${statusReason ? ` (${statusReason})` : ''}`);
@@ -378,13 +382,35 @@ export const createSessionGoalRuntime = ({
});
const structured = extractJsonObject(generated?.text);
const verdict = typeof structured?.verdict === 'string' ? structured.verdict.trim().toLowerCase() : '';
if (!['continue', 'complete', 'blocked'].includes(verdict)) return null;
if (!structured || !['continue', 'complete', 'blocked'].includes(verdict)) {
console.warn('[session-goal:diagnostic] audit parse failed', {
sessionId: lastAssistantInfo?.sessionID ?? null,
provider: generated?.providerID ?? null,
model: generated?.modelID ?? null,
outputChars: typeof generated?.text === 'string' ? generated.text.length : 0,
jsonObjectFound: Boolean(structured),
verdict: verdict || null,
});
return null;
}
console.log('[session-goal:diagnostic] audit verdict', {
sessionId: lastAssistantInfo?.sessionID ?? null,
provider: generated?.providerID ?? null,
model: generated?.modelID ?? null,
outputChars: generated.text.length,
verdict,
});
let note = clampText(structured?.note, NOTE_CHAR_LIMIT);
if (note && hasScriptMismatch(note, `${goal.objective}\n${assistantText}`)) {
console.warn('[session-goal] dropped audit note: language mismatch with objective');
note = '';
}
return { verdict, note };
return {
verdict,
note,
evaluationProviderID: generated.providerID,
evaluationModelID: generated.modelID,
};
} catch (error) {
// No authenticated small model (404) or a transient failure — the loop
// still terminates via markers, budget, and the turn cap.
@@ -656,15 +682,22 @@ export const createSessionGoalRuntime = ({
if (audit?.verdict === 'complete') {
await settleGoal({
sessionId, directory, goal, status: 'complete', statusReason: 'verified by audit', note: audit.note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID,
evaluationProviderID: audit.evaluationProviderID, evaluationModelID: audit.evaluationModelID,
});
return;
}
if (audit?.verdict === 'blocked') {
blockedStreak = goal.blockedStreak + 1;
console.warn('[session-goal:diagnostic] blocked audit streak', {
sessionId,
blockedStreak,
blockedStreakLimit: BLOCKED_STREAK_LIMIT,
});
if (blockedStreak >= BLOCKED_STREAK_LIMIT) {
await settleGoal({
sessionId, directory, goal, status: 'blocked', statusReason: audit.note || 'blocked per audit', note: audit.note, tokensUsed, tokensBaseline, tokensCommitted, lastAccountedMessageID,
evaluationProviderID: audit.evaluationProviderID, evaluationModelID: audit.evaluationModelID,
});
return;
}
@@ -684,6 +717,8 @@ export const createSessionGoalRuntime = ({
auditFailStreak,
statusReason: '',
...(audit?.note ? { note: audit.note } : {}),
...(audit?.evaluationProviderID ? { evaluationProviderID: audit.evaluationProviderID } : {}),
...(audit?.evaluationModelID ? { evaluationModelID: audit.evaluationModelID } : {}),
}));
if (!written) {
console.log('[session-goal] goal changed during tick, dropping continuation');
packages/web/server/lib/session-goal/runtime.test.js
+11 -2
View File
@@ -123,7 +123,7 @@ describe('session goal live activity gate', () => {
const requests = [];
const fetchImpl = vi.fn(async (input, init = {}) => {
const pathname = requestPath(input);
requests.push({ pathname, method: init.method ?? 'GET' });
requests.push({ pathname, method: init.method ?? 'GET', body: init.body });
if (pathname === `/session/${SESSION_ID}` && init.method === 'PATCH') return jsonResponse(session);
if (pathname === `/session/${SESSION_ID}`) return jsonResponse(session);
if (pathname === '/session/status') return jsonResponse({});
@@ -147,6 +147,8 @@ describe('session goal live activity gate', () => {
const service = {
generateSmallModelText: vi.fn(async () => ({
text: '{"verdict":"complete","note":"Task verified complete"}',
providerID: 'provider',
modelID: 'model',
})),
};
vi.stubGlobal('fetch', fetchImpl);
@@ -164,7 +166,14 @@ describe('session goal live activity gate', () => {
await vi.advanceTimersByTimeAsync(10);
expect(service.generateSmallModelText).toHaveBeenCalledOnce();
expect(requests).toContainEqual({ pathname: `/session/${SESSION_ID}`, method: 'PATCH' });
const patch = requests.find((request) => request.pathname === `/session/${SESSION_ID}` && request.method === 'PATCH');
expect(patch).toBeDefined();
const writtenGoal = JSON.parse(patch.body).metadata.openchamber.goal;
expect(writtenGoal).toMatchObject({
status: 'complete',
evaluationProviderID: 'provider',
evaluationModelID: 'model',
});
runtime.stop();
});
});
packages/web/server/lib/small-model/DOCUMENTATION.md
+6 -1
View File
@@ -49,7 +49,12 @@ other runtime API.
- Everything else: OpenAI-compatible `/chat/completions` against the
provider's base URL, resolved from (1) `provider.<id>.options.baseURL`
in the OpenCode config, (2) the hardcoded `https://api.openai.com/v1`
endpoint, or (3) the provider's `api` field from the models.dev catalog.
endpoint, or (3) the provider's `api` field from the models.dev catalog.
- `[small-model:diagnostic]` logs record provider/model, input character
counts, output budget, thinking toggle, HTTP/finish status, and
content/reasoning lengths without logging prompts, response text, or
credentials. Goal audit parsing similarly emits
`[session-goal:diagnostic]` structural verdict metadata.
- `catalog.js` — models.dev catalog via the shared in-process cache
(`../opencode/models-metadata.js`, also serving
`/api/openchamber/models-metadata`).
packages/web/server/lib/small-model/call.js
+27
View File
@@ -104,6 +104,15 @@ const ensureFreshOpenaiOauth = async (entry) => {
const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system, maxOutputTokens, providerLabel, extraBody }) => {
const trimmedBase = baseURL.replace(/\/+$/, '');
console.log('[small-model:diagnostic] request', {
provider: providerLabel,
model: modelID,
maxOutputTokens,
thinkingDisabled: extraBody?.thinking?.type === 'disabled',
promptChars: prompt.length,
systemChars: system?.length ?? 0,
inputChars: prompt.length + (system?.length ?? 0),
});
const response = await fetch(`${trimmedBase}/chat/completions`, {
method: 'POST',
headers: {
@@ -123,11 +132,29 @@ const callOpenaiCompatible = async ({ baseURL, headers, modelID, prompt, system,
}),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
console.log('[small-model:diagnostic] response', {
provider: providerLabel,
model: modelID,
httpStatus: response.status,
ok: response.ok,
});
if (!response.ok) {
throw await httpError(response, providerLabel);
}
const payload = await response.json();
const message = payload?.choices?.[0]?.message;
console.log('[small-model:diagnostic] completion', {
provider: providerLabel,
model: modelID,
finishReason: payload?.choices?.[0]?.finish_reason ?? null,
contentType: Array.isArray(message?.content) ? 'parts' : typeof message?.content,
contentChars: typeof message?.content === 'string'
? message.content.length
: Array.isArray(message?.content)
? message.content.reduce((total, part) => total + (typeof part?.text === 'string' ? part.text.length : 0), 0)
: 0,
reasoningChars: typeof message?.reasoning_content === 'string' ? message.reasoning_content.length : 0,
});
// Providers disagree on the content shape: plain string, an array of
// typed parts, or (thinking models) an empty content with the budget spent