diff --git a/.github/scripts/eval-report.mjs b/.github/scripts/eval-report.mjs index 4c935f6..1933759 100644 --- a/.github/scripts/eval-report.mjs +++ b/.github/scripts/eval-report.mjs @@ -9,7 +9,12 @@ const stripTerminalControls = (text) => stripVTControlCharacters(String(text ?? // eslint-disable-next-line no-control-regex .replace(/[\u0000-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]+/gu, " ") .trim(); -const escapeCell = (text) => stripTerminalControls(text).replaceAll("|", "\\|").replaceAll("<", "<").replaceAll(">", ">"); +const escapeCell = (text) => stripTerminalControls(text) + .replaceAll("&", "&") + .replaceAll("|", "\\|") + .replaceAll("`", "`") + .replaceAll("<", "<") + .replaceAll(">", ">"); const finiteNonNegative = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; const sanitizeIdentifier = (value, fallback = "check") => { const cleaned = String(value ?? fallback).replace(/[^A-Za-z0-9._:-]+/g, "-").replace(/^-+|-+$/g, ""); @@ -278,6 +283,8 @@ export function summarizeRun(records) { group.failures.add(sanitizeGraderId(detail)); } } + group.gradersTotal = (group.gradersTotal ?? 0) + (grade?.details?.length ?? 0); + group.gradersPassed = (group.gradersPassed ?? 0) + (grade?.details ?? []).filter((d) => d.passed).length; } if (!trialCount) throw new Error("No cases were evaluated."); @@ -306,6 +313,8 @@ export function summarizeRun(records) { : {}), } : undefined, + gradersPassed: group.gradersPassed ?? 0, + gradersTotal: group.gradersTotal ?? 0, failures: [...group.failures], }; }); @@ -505,11 +514,190 @@ function formatDurationDelta(deltaMs, pct) { return `${sign}${formatDuration(Math.abs(deltaMs))}${pctText}`; } +export function summarizeSkillVerdicts(comparison) { + const bySkill = new Map(); + for (const item of comparison.cases) { + const skillName = item.candidate?.evalName ?? item.baseline?.evalName ?? "eval"; + let group = bySkill.get(skillName); + if (!group) { + group = { + skillName, + cases: [], + fixed: 0, + broke: 0, + errored: 0, + changed: false, + basePassed: 0, + baseTotal: 0, + candPassed: 0, + candTotal: 0, + baseOutputTokens: 0, + candOutputTokens: 0, + hasOutputTokens: true, + }; + bySkill.set(skillName, group); + } + group.cases.push(item); + if (item.baseline) { + group.baseTotal++; + if (item.baseline.passed) group.basePassed++; + } + if (item.candidate) { + group.candTotal++; + if (item.candidate.passed) group.candPassed++; + } + const baseOut = item.baseline?.tokenUsage?.outputTokens; + const candOut = item.candidate?.tokenUsage?.outputTokens; + const sameCaseTrials = Boolean( + item.baseline && item.candidate && (item.baseline.trials ?? 1) === (item.candidate.trials ?? 1), + ); + if (sameCaseTrials && typeof baseOut === "number" && typeof candOut === "number") { + group.baseOutputTokens += baseOut; + group.candOutputTokens += candOut; + } else { + group.hasOutputTokens = false; + } + if (item.candidate?.status === "ERROR / NOT EVALUATED" || !item.candidate) { + group.errored++; + group.changed = true; + } else if (item.transition === "Fixed") { + group.fixed++; + group.changed = true; + } else if (item.transition === "Regressed") { + group.broke++; + group.changed = true; + } else if ( + item.transition === "Improved score" + || item.transition === "Added" + || item.transition === "Removed" + || Math.abs(item.outputTokenDeltaPct ?? 0) >= 10 + ) { + group.changed = true; + } + } + + const severityOrder = { "๐Ÿ›‘ No ship": 0, "โš ๏ธ Re-run": 1, "โœ… Ship": 2 }; + return [...bySkill.values()] + .map((group) => { + const hasBehavioralFailure = group.cases.some((c) => c.candidate?.status === "BEHAVIORAL FAILURE"); + const verdict = group.candTotal === 0 || (group.errored > 0 && !hasBehavioralFailure && group.broke === 0) + ? "โš ๏ธ Re-run" + : group.candPassed === group.candTotal && group.broke === 0 && group.errored === 0 + ? "โœ… Ship" + : "๐Ÿ›‘ No ship"; + const reasons = []; + if (group.fixed > 0 && group.broke > 0) { + reasons.push(`${group.broke} ${group.broke === 1 ? "scenario" : "scenarios"} broke, ${group.fixed} fixed (${group.basePassed}/${group.baseTotal} โ†’ ${group.candPassed}/${group.candTotal} passing)`); + } else if (group.broke > 0) { + reasons.push(`${group.broke} ${group.broke === 1 ? "scenario" : "scenarios"} broke (${group.basePassed}/${group.baseTotal} โ†’ ${group.candPassed}/${group.candTotal} passing)`); + } else if (group.errored > 0) { + reasons.push(`${group.errored} ${group.errored === 1 ? "scenario" : "scenarios"} errored (${group.basePassed}/${group.baseTotal} โ†’ ${group.candPassed}/${group.candTotal} passing)`); + } else if (group.fixed > 0) { + reasons.push(`${group.fixed} ${group.fixed === 1 ? "scenario" : "scenarios"} fixed (${group.basePassed}/${group.baseTotal} โ†’ ${group.candPassed}/${group.candTotal} passing)`); + } else { + reasons.push(`${group.candPassed}/${group.candTotal} scenarios passing`); + } + if (group.hasOutputTokens && group.baseOutputTokens > 0) { + const outDelta = group.candOutputTokens - group.baseOutputTokens; + const outPct = (outDelta / group.baseOutputTokens) * 100; + if (Math.abs(outPct) >= 0.1) { + const sign = outPct > 0 ? "+" : ""; + reasons.push(`${sign}${outPct.toFixed(1)}% output tokens`); + } + } + const summaryCounts = []; + if (group.broke > 0) summaryCounts.push(`${group.broke} broke`); + if (group.errored > 0) summaryCounts.push(`${group.errored} errored`); + if (group.fixed > 0) summaryCounts.push(`${group.fixed} fixed`); + if (summaryCounts.length === 0) summaryCounts.push(`${group.candPassed}/${group.candTotal} passing`); + return { + ...group, + verdict, + summaryLabel: `${summaryCounts.join(", ")} (${group.basePassed}/${group.baseTotal} โ†’ ${group.candPassed}/${group.candTotal} passing)`, + why: reasons.join(" ยท "), + }; + }) + .sort((a, b) => (severityOrder[a.verdict] ?? 3) - (severityOrder[b.verdict] ?? 3)); +} + +function describeCaseChange(item) { + const baseFailures = item.baseline?.failures ?? []; + const candFailures = item.candidate?.failures ?? []; + if (item.candidate?.status === "ERROR / NOT EVALUATED") { + return "Trial execution error; re-run workflow to verify"; + } + const hasCounts = Boolean(item.baseline?.gradersTotal && item.candidate?.gradersTotal); + const countPrefix = hasCounts + ? `${item.baseline.gradersPassed}/${item.baseline.gradersTotal} โ†’ ${item.candidate.gradersPassed}/${item.candidate.gradersTotal} checks passed` + : undefined; + if (item.transition === "Fixed") { + const resolved = baseFailures.filter((f) => !candFailures.includes(f)); + const prefix = countPrefix ?? "Now passes all checks"; + return resolved.length + ? `${prefix} (fixed: ${resolved.map((f) => `\`${escapeCell(f)}\``).join(", ")})` + : prefix; + } + if (item.transition === "Regressed") { + const newlyBroken = candFailures.filter((f) => !baseFailures.includes(f)); + const prefix = countPrefix ?? "Check failed"; + return newlyBroken.length + ? `${prefix} (failed: ${newlyBroken.map((f) => `\`${escapeCell(f)}\``).join(", ")})` + : prefix; + } + if (item.candidate?.passed) { + return item.candidate.gradersTotal + ? `Unchanged (${item.candidate.gradersPassed}/${item.candidate.gradersTotal} checks passed)` + : "Unchanged and passing"; + } + return candFailures.length + ? `Still failing (${candFailures.map((f) => `\`${escapeCell(f)}\``).join(", ")})` + : "Unchanged"; +} + +function renderSkillDetailsBlock(skill, defaultModel) { + const verdictEmoji = skill.verdict.split(" ")[0]; + const modelLabel = defaultModel || "default"; + return [ + "
", + `${verdictEmoji} ${escapeCell(skill.skillName)} โ€” ${skill.summaryLabel}`, + "", + "| | Scenario | Model | Score | Output Tokens | What happened |", + "| --- | --- | --- | --- | ---: | --- |", + ...skill.cases.map((item) => { + const badge = item.candidate?.status === "ERROR / NOT EVALUATED" + ? "โš ๏ธ Error" + : item.transition === "Fixed" + ? "๐Ÿ“ˆ Fixed" + : item.transition === "Regressed" + ? "๐Ÿ“‰ Broke" + : item.candidate?.passed + ? "โœ… Passed" + : "โŒ Failed"; + const rawCaseName = item.candidate?.name ?? item.baseline?.name ?? item.key; + const model = item.candidate?.models?.[0] ?? item.baseline?.models?.[0] ?? modelLabel; + const scoreText = `${formatScore(item.baseline?.score)} โ†’ ${item.candidate?.status === "ERROR / NOT EVALUATED" ? "โ€”" : formatScore(item.candidate?.score)}`; + const baseOut = item.baseline?.tokenUsage?.outputTokens; + const candOut = item.candidate?.tokenUsage?.outputTokens; + const outText = typeof baseOut === "number" && typeof candOut === "number" && item.candidate?.status !== "ERROR / NOT EVALUATED" + ? `${baseOut.toLocaleString("en-US")} โ†’ ${candOut.toLocaleString("en-US")}${typeof item.outputTokenDelta === "number" ? ` (${formatTokenDelta(item.outputTokenDelta, item.outputTokenDeltaPct)})` : ""}` + : typeof baseOut === "number" + ? `${baseOut.toLocaleString("en-US")} โ†’ โ€”` + : "โ€”"; + return `| ${badge} | ${escapeCell(rawCaseName)} | ${escapeCell(model)} | ${scoreText} | ${outText} | ${describeCaseChange(item)} |`; + }), + "", + `Pass rate (before โ†’ after): ${escapeCell(modelLabel)} ${skill.basePassed}/${skill.baseTotal} โ†’ ${skill.candPassed}/${skill.candTotal}`, + "", + "
", + "", + ]; +} + export function renderReport(summary, path, { baselineSummary, baselinePath } = {}) { const modelHeader = summary.models?.length ? ` | Model: \`${escapeCell(summary.models.join(", "))}\`` : ""; - const lines = [ + const singleRunLines = [ "## Behavioral eval report", "", `Saved run: \`${escapeCell(path)}\`${modelHeader}`, @@ -533,41 +721,119 @@ export function renderReport(summary, path, { baselineSummary, baselinePath } = row.failures.map((grader) => `- ${escapeCell(row.name)} / ${escapeCell(sanitizeGraderId(grader))}: check failed`)); const errorLines = summary.errors.map((error) => `- ERROR: ${escapeCell(error)}`); if (failureLines.length || errorLines.length) { - lines.push(...failureLines, ...errorLines, ""); + singleRunLines.push(...failureLines, ...errorLines, ""); + } + + if (!baselineSummary) { + return singleRunLines.join("\n"); } - if (baselineSummary) { - const comparison = compareSummaries(baselineSummary, summary); + const comparison = compareSummaries(baselineSummary, summary); + const skillVerdicts = summarizeSkillVerdicts(comparison); + const hasNoShip = skillVerdicts.some((s) => s.verdict === "๐Ÿ›‘ No ship") || summary.failedCount > 0; + const hasRerun = skillVerdicts.some((s) => s.verdict === "โš ๏ธ Re-run") + || summary.errorCount > 0 + || summary.errors.length > 0 + || baselineSummary.errors.length > 0; + const overallVerdict = hasNoShip ? "๐Ÿ›‘ No ship" : hasRerun ? "โš ๏ธ Re-run" : "โœ… Ship"; + const baseClean = (baselineSummary.failedCount + baselineSummary.errorCount) === 0 && baselineSummary.errors.length === 0; + const candClean = (summary.failedCount + summary.errorCount) === 0 && summary.errors.length === 0; + const baseOutcome = baseClean ? "passed" : "failed"; + const candOutcome = overallVerdict === "โœ… Ship" + ? "passed (โœ… Ship)" + : overallVerdict === "๐Ÿ›‘ No ship" + ? "failed (๐Ÿ›‘ No ship)" + : `${candClean ? "passed" : "failed"} (โš ๏ธ Re-run)`; + const outcomeDelta = overallVerdict === "โœ… Ship" + ? (baseOutcome === "failed" ? "fixed" : "unchanged") + : overallVerdict === "๐Ÿ›‘ No ship" + ? (baseOutcome === "passed" || skillVerdicts.some((s) => s.broke > 0) ? "regressed" : "๐Ÿ›‘ No ship") + : "โš ๏ธ Re-run"; + const defaultModel = summary.models?.[0] ?? baselineSummary.models?.[0] ?? "default"; + + const actionableSkills = skillVerdicts.filter((s) => s.verdict !== "โœ… Ship"); + const passingChangedSkills = skillVerdicts.filter((s) => s.verdict === "โœ… Ship" && (s.changed || skillVerdicts.length === 1)); + const unchangedPassingSkills = skillVerdicts.filter((s) => s.verdict === "โœ… Ship" && !s.changed && skillVerdicts.length > 1); + + const lines = [ + `## Evals: ${overallVerdict}`, + "", + "| Skill | Verdict | Why |", + "| --- | --- | --- |", + ...skillVerdicts.map((s) => { + const boldVerdict = s.verdict.replace(/^(โœ…|๐Ÿ›‘|โš ๏ธ)\s+(.*)$/u, "$1 **$2**"); + return `| **${escapeCell(s.skillName)}** | ${boldVerdict} | ${s.why} |`; + }), + "", + ]; + + for (const skill of actionableSkills) { + lines.push(...renderSkillDetailsBlock(skill, defaultModel)); + } + + if (actionableSkills.length > 0) { + lines.push("---", ""); + } + + for (const skill of passingChangedSkills) { + lines.push(...renderSkillDetailsBlock(skill, defaultModel)); + } + + if (unchangedPassingSkills.length > 0) { + const unchangedScenarios = unchangedPassingSkills.reduce((sum, s) => sum + s.candTotal, 0); lines.push( - "## Baseline vs. Candidate Comparison", + "
", + `๐Ÿ“‹ Unchanged evals & run configuration โ€” ${unchangedPassingSkills.length} passing (${unchangedScenarios}/${unchangedScenarios} scenarios)`, "", - ...(baselinePath ? [`Baseline: \`${escapeCell(baselinePath)}\` | Candidate: \`${escapeCell(path)}\``, ""] : []), - "| Metric | Baseline | Candidate | Delta |", + "| Skill | Status | Scenarios (Before โ†’ After) | Details |", "| --- | --- | --- | --- |", - `| Passed | ${baselineSummary.passedCount} | ${summary.passedCount} | ${formatSignedInt(comparison.deltas.passed)} |`, - `| Failed | ${baselineSummary.failedCount + baselineSummary.errorCount} | ${summary.failedCount + summary.errorCount} | ${formatSignedInt(comparison.deltas.failed)} |`, - `| Avg Score | ${formatScore(baselineSummary.averageScore)} | ${formatScore(summary.averageScore)} | ${formatScoreDelta(comparison.deltas.averageScore)} |`, - `| Total Turns | ${baselineSummary.totalTurns ?? "โ€”"} | ${summary.totalTurns ?? "โ€”"} | ${formatSignedInt(comparison.deltas.totalTurns)} |`, - `| Total Tokens | ${formatTokens(baselineSummary.tokenUsage)} | ${formatTokens(summary.tokenUsage)} | ${formatTokenDelta(comparison.deltas.totalTokens, comparison.deltas.totalTokensPct)} |`, - ...(typeof baselineSummary.tokenUsage?.outputTokens === "number" && typeof summary.tokenUsage?.outputTokens === "number" - ? [`| Output Tokens | ${baselineSummary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${summary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${formatTokenDelta(comparison.deltas.outputTokens, comparison.deltas.outputTokensPct)} |`] - : []), - `| Agent Wall Time | ${formatDuration(baselineSummary.totalWallTimeMs)} | ${formatDuration(summary.totalWallTimeMs)} | ${formatDurationDelta(comparison.deltas.totalWallTimeMs)} |`, - `| Total Duration | ${formatDuration(baselineSummary.totalDurationMs)} | ${formatDuration(summary.totalDurationMs)} | ${formatDurationDelta(comparison.deltas.totalDurationMs)} |`, + ...unchangedPassingSkills.map((s) => + `| \`${escapeCell(s.skillName)}\` | โœ… Unchanged | ${s.basePassed}/${s.baseTotal} โ†’ ${s.candPassed}/${s.candTotal} | All scenarios passing on \`${escapeCell(defaultModel)}\` |`, + ), "", - "| Case | Baseline | Candidate | Score Delta | Turn Delta | Token Delta | Wall Time Delta | Transition |", - "| --- | --- | --- | --- | --- | --- | --- | --- |", - ...comparison.cases.map((item) => { - const baseCell = item.baseline ? `${item.baseline.status} (${formatScore(item.baseline.score)})` : "โ€”"; - const candCell = item.candidate ? `${item.candidate.status} (${formatScore(item.candidate.score)})` : "โ€”"; - return `| ${escapeCell(item.key)} | ${baseCell} | ${candCell} | ${formatScoreDelta(item.scoreDelta)} | ${formatSignedInt(item.turnsDelta)} | ${formatTokenDelta(item.tokenDelta, item.tokenDeltaPct)} | ${formatDurationDelta(item.wallTimeDeltaMs, item.wallTimeDeltaPct)} | ${item.transition} |`; - }), + `Run config: ${comparison.cases.length} scenarios across ${skillVerdicts.length} ${skillVerdicts.length === 1 ? "eval" : "evals"} ยท ${escapeCell(defaultModel)}`, "", - "> **Note:** Single-run score, token, and duration deltas are observed signals for review rather than multi-trial statistical proof.", + "
", "", ); } + lines.push( + "
", + `๐Ÿ“Š Telemetry, token deltas & run evidence โ€” ${skillVerdicts.length} ${skillVerdicts.length === 1 ? "skill" : "skills"} evaluated (${comparison.cases.length} ${comparison.cases.length === 1 ? "scenario" : "scenarios"})`, + "", + "## Baseline vs. Candidate Comparison", + "", + ...(baselinePath ? [`Baseline: \`${escapeCell(baselinePath)}\` | Candidate: \`${escapeCell(path)}\``, ""] : []), + "| Metric | Baseline | Candidate | Delta |", + "| --- | --- | --- | --- |", + `| Outcome | ${baseOutcome} | ${candOutcome} | ${outcomeDelta} |`, + `| Passed | ${baselineSummary.passedCount} | ${summary.passedCount} | ${formatSignedInt(comparison.deltas.passed)} |`, + `| Failed | ${baselineSummary.failedCount + baselineSummary.errorCount} | ${summary.failedCount + summary.errorCount} | ${formatSignedInt(comparison.deltas.failed)} |`, + `| Avg Score | ${formatScore(baselineSummary.averageScore)} | ${formatScore(summary.averageScore)} | ${formatScoreDelta(comparison.deltas.averageScore)} |`, + `| Total Turns | ${baselineSummary.totalTurns ?? "โ€”"} | ${summary.totalTurns ?? "โ€”"} | ${formatSignedInt(comparison.deltas.totalTurns)} |`, + `| Total Tokens | ${formatTokens(baselineSummary.tokenUsage)} | ${formatTokens(summary.tokenUsage)} | ${formatTokenDelta(comparison.deltas.totalTokens, comparison.deltas.totalTokensPct)} |`, + ...(typeof baselineSummary.tokenUsage?.outputTokens === "number" && typeof summary.tokenUsage?.outputTokens === "number" + ? [`| Output Tokens | ${baselineSummary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${summary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${formatTokenDelta(comparison.deltas.outputTokens, comparison.deltas.outputTokensPct)} |`] + : []), + `| Agent Wall Time | ${formatDuration(baselineSummary.totalWallTimeMs)} | ${formatDuration(summary.totalWallTimeMs)} | ${formatDurationDelta(comparison.deltas.totalWallTimeMs)} |`, + `| Total Duration | ${formatDuration(baselineSummary.totalDurationMs)} | ${formatDuration(summary.totalDurationMs)} | ${formatDurationDelta(comparison.deltas.totalDurationMs)} |`, + "", + "| Case | Baseline | Candidate | Score Delta | Turn Delta | Token Delta | Wall Time Delta | Transition |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ...comparison.cases.map((item) => { + const baseCell = item.baseline ? `${item.baseline.status} (${formatScore(item.baseline.score)})` : "โ€”"; + const candCell = item.candidate ? `${item.candidate.status} (${formatScore(item.candidate.score)})` : "โ€”"; + return `| ${escapeCell(item.key)} | ${baseCell} | ${candCell} | ${formatScoreDelta(item.scoreDelta)} | ${formatSignedInt(item.turnsDelta)} | ${formatTokenDelta(item.tokenDelta, item.tokenDeltaPct)} | ${formatDurationDelta(item.wallTimeDeltaMs, item.wallTimeDeltaPct)} | ${item.transition} |`; + }), + "", + "> **Note:** Single-run score, token, and duration deltas are observed signals for review rather than multi-trial statistical proof.", + "", + ...singleRunLines, + "
", + "", + ); + return lines.join("\n"); } diff --git a/.github/scripts/eval-report.test.mjs b/.github/scripts/eval-report.test.mjs index 125ae00..3204138 100644 --- a/.github/scripts/eval-report.test.mjs +++ b/.github/scripts/eval-report.test.mjs @@ -16,6 +16,7 @@ import { loadResults, renderReport, summarizeRun, + summarizeSkillVerdicts, } from "./eval-report.mjs"; const root = fileURLToPath(new URL("../../", import.meta.url)); @@ -255,6 +256,10 @@ test("compareSummaries and renderReport surface fixed failures, regressions, and baselineSummary: baseline, baselinePath: "baseline/results.jsonl", }); + assert.match(markdown, /## Evals: โœ… Ship/); + assert.match(markdown, /\| \*\*pr-description\*\* \| โœ… \*\*Ship\*\* \| 1 scenario fixed \(1\/2 โ†’ 2\/2 passing\) ยท -64\.0% output tokens \|/); + assert.match(markdown, /โœ… pr-description<\/strong> โ€” 1 fixed \(1\/2 โ†’ 2\/2 passing\)<\/summary>/); + assert.match(markdown, /๐Ÿ“Š Telemetry, token deltas & run evidence<\/strong>/); assert.match(markdown, /## Baseline vs\. Candidate Comparison/); assert.match(markdown, /\| Total Turns \| 3 \| 3 \| 0 \|/); assert.match(markdown, /\| Total Tokens \| 6,960 \(in: 5,570, out: 1,390\) \| 3,300 \(in: 2,800, out: 500\) \| -3,660 \(-52\.6%\) \|/); @@ -263,6 +268,43 @@ test("compareSummaries and renderReport surface fixed failures, regressions, and assert.match(markdown, /\| pr-description \/ pr-description-drafts-structured-pr-body \| BEHAVIORAL FAILURE \(50\.0%\) \| PASS \(100\.0%\) \| \+50\.0% \| 0 \| -2,410 \(-54\.8%\) \| -2\.4s \(-46\.2%\) \| Fixed \|/); assert.match(markdown, /\| pr-description \/ pr-description-ignores-bug-and-review-question \| PASS \(100\.0%\) \| PASS \(100\.0%\) \| 0\.0% \| 0 \| -1,250 \(-48\.8%\) \| -1\.3s \(-41\.9%\) \| Reduced tokens at parity \|/); assert.match(markdown, /Single-run score, token, and duration deltas are observed signals for review/); + + // Multi-skill mixed verdict ordering: No ship and Re-run above ---, Ship and Unchanged/Telemetry below --- + const multiSkillBase = summarizeRun([ + makeTrial({ runId: "ms-base", evalName: "pr-description", stimulus: "drafts-body", passed: false, gradeScore: 0.5, outputTokens: 900, inputTokens: 2000, totalTokens: 2900, details: [{ name: "program", passed: false, score: 0 }, { name: "completed", passed: true, score: 1 }] }), + makeTrial({ runId: "ms-base", evalName: "release-plugins", stimulus: "preserves-version", passed: true, gradeScore: 1, outputTokens: 400, inputTokens: 2000, totalTokens: 2400, details: [{ name: "skill-invocation", passed: true, score: 1 }] }), + makeTrial({ runId: "ms-base", evalName: "try-plugin-locally", stimulus: "prepares-session", passed: true, gradeScore: 1, outputTokens: 500, inputTokens: 2000, totalTokens: 2500 }), + makeTrial({ runId: "ms-base", evalName: "update-remote-pins", stimulus: "updates-pins", passed: true, gradeScore: 1, outputTokens: 300, inputTokens: 1500, totalTokens: 1800 }), + makeFooter("ms-base"), + ]); + const multiSkillCand = summarizeRun([ + makeTrial({ runId: "ms-cand", evalName: "pr-description", stimulus: "drafts-body", passed: true, gradeScore: 1, outputTokens: 400, inputTokens: 2000, totalTokens: 2400, details: [{ name: "program", passed: true, score: 1 }, { name: "completed", passed: true, score: 1 }] }), + makeTrial({ runId: "ms-cand", evalName: "release-plugins", stimulus: "preserves-version", passed: false, gradeScore: 0, outputTokens: 390, inputTokens: 2000, totalTokens: 2390, details: [{ name: "skill-invocation", passed: false, score: 0 }] }), + makeTrial({ runId: "ms-cand", evalName: "try-plugin-locally", stimulus: "prepares-session", status: "error", passed: false, gradeScore: 0, outputTokens: 0, inputTokens: 0, totalTokens: 0 }), + makeTrial({ runId: "ms-cand", evalName: "update-remote-pins", stimulus: "updates-pins", passed: true, gradeScore: 1, outputTokens: 300, inputTokens: 1500, totalTokens: 1800 }), + makeFooter("ms-cand", true), + ]); + const multiSkillComparison = compareSummaries(multiSkillBase, multiSkillCand); + const verdicts = summarizeSkillVerdicts(multiSkillComparison); + assert.deepEqual(verdicts.map((v) => [v.skillName, v.verdict]), [ + ["release-plugins", "๐Ÿ›‘ No ship"], + ["try-plugin-locally", "โš ๏ธ Re-run"], + ["pr-description", "โœ… Ship"], + ["update-remote-pins", "โœ… Ship"], + ]); + const multiSkillMarkdown = renderReport(multiSkillCand, "cand/results.jsonl", { + baselineSummary: multiSkillBase, + baselinePath: "base/results.jsonl", + }); + assert.match(multiSkillMarkdown, /## Evals: ๐Ÿ›‘ No ship/); + const hrIndex = multiSkillMarkdown.indexOf("\n---\n"); + assert.ok(hrIndex > 0, "expected --- separator between actionable and passing sections"); + assert.ok(multiSkillMarkdown.indexOf("๐Ÿ›‘ release-plugins") < hrIndex); + assert.ok(multiSkillMarkdown.indexOf("โš ๏ธ try-plugin-locally") < hrIndex); + assert.ok(multiSkillMarkdown.indexOf("โœ… pr-description") > hrIndex); + assert.ok(multiSkillMarkdown.indexOf("๐Ÿ“‹ Unchanged evals & run configuration") > hrIndex); + assert.ok(multiSkillMarkdown.indexOf("๐Ÿ“Š Telemetry, token deltas & run evidence") > hrIndex); + assert.match(multiSkillMarkdown, /\| Outcome \| failed \| failed \(๐Ÿ›‘ No ship\) \|/); }); prDescriptionTest("eval-report CLI compares the shared pr-description baseline and candidate fixtures without writing GITHUB_STEP_SUMMARY", async (t) => { @@ -442,6 +484,19 @@ test("eval-report redacts raw grader evidence and error strings, requires comple assert.match(mismatchedTrialsReport, /\| Total Tokens \| 1,100 \(in: 900, out: 200\) \| 400 \(in: 300, out: 100\) \| โ€” \|/); assert.match(mismatchedTrialsReport, /\| Agent Wall Time \| 2\.2s \| 900ms \| โ€” \|/); assert.match(mismatchedTrialsReport, /\| Total Duration \| 2\.2s \| 900ms \| โ€” \|/); + assert.doesNotMatch(mismatchedTrialsReport, /output tokens/); + + // Footer execution error without trial failure still triggers Re-run and Outcome alignment + const footerErrCand = summarizeRun([ + makeTrial({ runId: "f-cand", stimulus: "repeat-case", passed: true, gradeScore: 1, durationMs: 900, inputTokens: 300, outputTokens: 100, totalTokens: 400 }), + makeFooter("f-cand", true), + ]); + const footerErrReport = renderReport(footerErrCand, "cand/results.jsonl", { + baselineSummary: singleTrialCand, + baselinePath: "base/results.jsonl", + }); + assert.match(footerErrReport, /## Evals: โš ๏ธ Re-run/); + assert.match(footerErrReport, /\| Outcome \| passed \| failed \(โš ๏ธ Re-run\) \| โš ๏ธ Re-run \|/); const extraCaseBase = summarizeRun([ makeTrial({ runId: "e-base", stimulus: "repeat-case", passed: true, gradeScore: 1, durationMs: 900, inputTokens: 300, outputTokens: 100, totalTokens: 400 }), diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 0a54480..a512d63 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -308,15 +308,18 @@ matching `evals//skills//eval.yaml` changes: 3. **Post sticky PR comment (`report`):** run `github/evals-action` with `command: report`, `report-id: evals`, and `baseline-report-id: evals-baseline` in a separate `pull-requests: write` - job to publish one sticky PR comment with the **Baseline vs. Candidate - comparison** table (Outcome, Passed/Failed, Score delta, Duration delta, and - Token `in/out` + `%` delta) alongside per-eval results. + job to publish one sticky PR comment with a top-level **Ship / No-ship / + Re-run** verdict summary table and a collapsible **Telemetry, token deltas & + run evidence** `
` section (`Outcome`, `Passed/Failed`, `Score + delta`, `Turns delta`, `Duration delta`, and `Token in/out + %` delta). > **Action version note:** `baseline-report-id` (`evals-baseline`) is provided > by [github/evals-action#34](https://github.com/github/evals-action/pull/34) -> (verified commit `3c551330898c1467b8ddf9ef7c7f6e6c6af489d6`). Pin -> `github/evals-action@3c551330898c1467b8ddf9ef7c7f6e6c6af489d6` until `#34` -> is merged and published to `github/evals-action@v0`. +> (`3c551330898c1467b8ddf9ef7c7f6e6c6af489d6`), and the top-level **Ship / +> No-ship** verdict table layout is added in +> [github/evals-action#35](https://github.com/github/evals-action/pull/35). Pin +> `github/evals-action` to that commit until published to +> `github/evals-action@v0`. ```yaml name: Skill eval comparison