diff --git a/.github/scripts/eval-report.mjs b/.github/scripts/eval-report.mjs
index 4c935f6..1933759 100644
--- a/.github/scripts/eval-report.mjs
+++ b/.github/scripts/eval-report.mjs
@@ -9,7 +9,12 @@ const stripTerminalControls = (text) => stripVTControlCharacters(String(text ??
// eslint-disable-next-line no-control-regex
.replace(/[\u0000-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]+/gu, " ")
.trim();
-const escapeCell = (text) => stripTerminalControls(text).replaceAll("|", "\\|").replaceAll("<", "<").replaceAll(">", ">");
+const escapeCell = (text) => stripTerminalControls(text)
+ .replaceAll("&", "&")
+ .replaceAll("|", "\\|")
+ .replaceAll("`", "`")
+ .replaceAll("<", "<")
+ .replaceAll(">", ">");
const finiteNonNegative = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
const sanitizeIdentifier = (value, fallback = "check") => {
const cleaned = String(value ?? fallback).replace(/[^A-Za-z0-9._:-]+/g, "-").replace(/^-+|-+$/g, "");
@@ -278,6 +283,8 @@ export function summarizeRun(records) {
group.failures.add(sanitizeGraderId(detail));
}
}
+ group.gradersTotal = (group.gradersTotal ?? 0) + (grade?.details?.length ?? 0);
+ group.gradersPassed = (group.gradersPassed ?? 0) + (grade?.details ?? []).filter((d) => d.passed).length;
}
if (!trialCount) throw new Error("No cases were evaluated.");
@@ -306,6 +313,8 @@ export function summarizeRun(records) {
: {}),
}
: undefined,
+ gradersPassed: group.gradersPassed ?? 0,
+ gradersTotal: group.gradersTotal ?? 0,
failures: [...group.failures],
};
});
@@ -505,11 +514,190 @@ function formatDurationDelta(deltaMs, pct) {
return `${sign}${formatDuration(Math.abs(deltaMs))}${pctText}`;
}
+export function summarizeSkillVerdicts(comparison) {
+ const bySkill = new Map();
+ for (const item of comparison.cases) {
+ const skillName = item.candidate?.evalName ?? item.baseline?.evalName ?? "eval";
+ let group = bySkill.get(skillName);
+ if (!group) {
+ group = {
+ skillName,
+ cases: [],
+ fixed: 0,
+ broke: 0,
+ errored: 0,
+ changed: false,
+ basePassed: 0,
+ baseTotal: 0,
+ candPassed: 0,
+ candTotal: 0,
+ baseOutputTokens: 0,
+ candOutputTokens: 0,
+ hasOutputTokens: true,
+ };
+ bySkill.set(skillName, group);
+ }
+ group.cases.push(item);
+ if (item.baseline) {
+ group.baseTotal++;
+ if (item.baseline.passed) group.basePassed++;
+ }
+ if (item.candidate) {
+ group.candTotal++;
+ if (item.candidate.passed) group.candPassed++;
+ }
+ const baseOut = item.baseline?.tokenUsage?.outputTokens;
+ const candOut = item.candidate?.tokenUsage?.outputTokens;
+ const sameCaseTrials = Boolean(
+ item.baseline && item.candidate && (item.baseline.trials ?? 1) === (item.candidate.trials ?? 1),
+ );
+ if (sameCaseTrials && typeof baseOut === "number" && typeof candOut === "number") {
+ group.baseOutputTokens += baseOut;
+ group.candOutputTokens += candOut;
+ } else {
+ group.hasOutputTokens = false;
+ }
+ if (item.candidate?.status === "ERROR / NOT EVALUATED" || !item.candidate) {
+ group.errored++;
+ group.changed = true;
+ } else if (item.transition === "Fixed") {
+ group.fixed++;
+ group.changed = true;
+ } else if (item.transition === "Regressed") {
+ group.broke++;
+ group.changed = true;
+ } else if (
+ item.transition === "Improved score"
+ || item.transition === "Added"
+ || item.transition === "Removed"
+ || Math.abs(item.outputTokenDeltaPct ?? 0) >= 10
+ ) {
+ group.changed = true;
+ }
+ }
+
+ const severityOrder = { "๐ No ship": 0, "โ ๏ธ Re-run": 1, "โ
Ship": 2 };
+ return [...bySkill.values()]
+ .map((group) => {
+ const hasBehavioralFailure = group.cases.some((c) => c.candidate?.status === "BEHAVIORAL FAILURE");
+ const verdict = group.candTotal === 0 || (group.errored > 0 && !hasBehavioralFailure && group.broke === 0)
+ ? "โ ๏ธ Re-run"
+ : group.candPassed === group.candTotal && group.broke === 0 && group.errored === 0
+ ? "โ
Ship"
+ : "๐ No ship";
+ const reasons = [];
+ if (group.fixed > 0 && group.broke > 0) {
+ reasons.push(`${group.broke} ${group.broke === 1 ? "scenario" : "scenarios"} broke, ${group.fixed} fixed (${group.basePassed}/${group.baseTotal} โ ${group.candPassed}/${group.candTotal} passing)`);
+ } else if (group.broke > 0) {
+ reasons.push(`${group.broke} ${group.broke === 1 ? "scenario" : "scenarios"} broke (${group.basePassed}/${group.baseTotal} โ ${group.candPassed}/${group.candTotal} passing)`);
+ } else if (group.errored > 0) {
+ reasons.push(`${group.errored} ${group.errored === 1 ? "scenario" : "scenarios"} errored (${group.basePassed}/${group.baseTotal} โ ${group.candPassed}/${group.candTotal} passing)`);
+ } else if (group.fixed > 0) {
+ reasons.push(`${group.fixed} ${group.fixed === 1 ? "scenario" : "scenarios"} fixed (${group.basePassed}/${group.baseTotal} โ ${group.candPassed}/${group.candTotal} passing)`);
+ } else {
+ reasons.push(`${group.candPassed}/${group.candTotal} scenarios passing`);
+ }
+ if (group.hasOutputTokens && group.baseOutputTokens > 0) {
+ const outDelta = group.candOutputTokens - group.baseOutputTokens;
+ const outPct = (outDelta / group.baseOutputTokens) * 100;
+ if (Math.abs(outPct) >= 0.1) {
+ const sign = outPct > 0 ? "+" : "";
+ reasons.push(`${sign}${outPct.toFixed(1)}% output tokens`);
+ }
+ }
+ const summaryCounts = [];
+ if (group.broke > 0) summaryCounts.push(`${group.broke} broke`);
+ if (group.errored > 0) summaryCounts.push(`${group.errored} errored`);
+ if (group.fixed > 0) summaryCounts.push(`${group.fixed} fixed`);
+ if (summaryCounts.length === 0) summaryCounts.push(`${group.candPassed}/${group.candTotal} passing`);
+ return {
+ ...group,
+ verdict,
+ summaryLabel: `${summaryCounts.join(", ")} (${group.basePassed}/${group.baseTotal} โ ${group.candPassed}/${group.candTotal} passing)`,
+ why: reasons.join(" ยท "),
+ };
+ })
+ .sort((a, b) => (severityOrder[a.verdict] ?? 3) - (severityOrder[b.verdict] ?? 3));
+}
+
+function describeCaseChange(item) {
+ const baseFailures = item.baseline?.failures ?? [];
+ const candFailures = item.candidate?.failures ?? [];
+ if (item.candidate?.status === "ERROR / NOT EVALUATED") {
+ return "Trial execution error; re-run workflow to verify";
+ }
+ const hasCounts = Boolean(item.baseline?.gradersTotal && item.candidate?.gradersTotal);
+ const countPrefix = hasCounts
+ ? `${item.baseline.gradersPassed}/${item.baseline.gradersTotal} โ ${item.candidate.gradersPassed}/${item.candidate.gradersTotal} checks passed`
+ : undefined;
+ if (item.transition === "Fixed") {
+ const resolved = baseFailures.filter((f) => !candFailures.includes(f));
+ const prefix = countPrefix ?? "Now passes all checks";
+ return resolved.length
+ ? `${prefix} (fixed: ${resolved.map((f) => `\`${escapeCell(f)}\``).join(", ")})`
+ : prefix;
+ }
+ if (item.transition === "Regressed") {
+ const newlyBroken = candFailures.filter((f) => !baseFailures.includes(f));
+ const prefix = countPrefix ?? "Check failed";
+ return newlyBroken.length
+ ? `${prefix} (failed: ${newlyBroken.map((f) => `\`${escapeCell(f)}\``).join(", ")})`
+ : prefix;
+ }
+ if (item.candidate?.passed) {
+ return item.candidate.gradersTotal
+ ? `Unchanged (${item.candidate.gradersPassed}/${item.candidate.gradersTotal} checks passed)`
+ : "Unchanged and passing";
+ }
+ return candFailures.length
+ ? `Still failing (${candFailures.map((f) => `\`${escapeCell(f)}\``).join(", ")})`
+ : "Unchanged";
+}
+
+function renderSkillDetailsBlock(skill, defaultModel) {
+ const verdictEmoji = skill.verdict.split(" ")[0];
+ const modelLabel = defaultModel || "default";
+ return [
+ "",
+ `${verdictEmoji} ${escapeCell(skill.skillName)} โ ${skill.summaryLabel}
`,
+ "",
+ "| | Scenario | Model | Score | Output Tokens | What happened |",
+ "| --- | --- | --- | --- | ---: | --- |",
+ ...skill.cases.map((item) => {
+ const badge = item.candidate?.status === "ERROR / NOT EVALUATED"
+ ? "โ ๏ธ Error"
+ : item.transition === "Fixed"
+ ? "๐ Fixed"
+ : item.transition === "Regressed"
+ ? "๐ Broke"
+ : item.candidate?.passed
+ ? "โ
Passed"
+ : "โ Failed";
+ const rawCaseName = item.candidate?.name ?? item.baseline?.name ?? item.key;
+ const model = item.candidate?.models?.[0] ?? item.baseline?.models?.[0] ?? modelLabel;
+ const scoreText = `${formatScore(item.baseline?.score)} โ ${item.candidate?.status === "ERROR / NOT EVALUATED" ? "โ" : formatScore(item.candidate?.score)}`;
+ const baseOut = item.baseline?.tokenUsage?.outputTokens;
+ const candOut = item.candidate?.tokenUsage?.outputTokens;
+ const outText = typeof baseOut === "number" && typeof candOut === "number" && item.candidate?.status !== "ERROR / NOT EVALUATED"
+ ? `${baseOut.toLocaleString("en-US")} โ ${candOut.toLocaleString("en-US")}${typeof item.outputTokenDelta === "number" ? ` (${formatTokenDelta(item.outputTokenDelta, item.outputTokenDeltaPct)})` : ""}`
+ : typeof baseOut === "number"
+ ? `${baseOut.toLocaleString("en-US")} โ โ`
+ : "โ";
+ return `| ${badge} | ${escapeCell(rawCaseName)} | ${escapeCell(model)} | ${scoreText} | ${outText} | ${describeCaseChange(item)} |`;
+ }),
+ "",
+ `Pass rate (before โ after): ${escapeCell(modelLabel)} ${skill.basePassed}/${skill.baseTotal} โ ${skill.candPassed}/${skill.candTotal}`,
+ "",
+ " ",
+ "",
+ ];
+}
+
export function renderReport(summary, path, { baselineSummary, baselinePath } = {}) {
const modelHeader = summary.models?.length
? ` | Model: \`${escapeCell(summary.models.join(", "))}\``
: "";
- const lines = [
+ const singleRunLines = [
"## Behavioral eval report",
"",
`Saved run: \`${escapeCell(path)}\`${modelHeader}`,
@@ -533,41 +721,119 @@ export function renderReport(summary, path, { baselineSummary, baselinePath } =
row.failures.map((grader) => `- ${escapeCell(row.name)} / ${escapeCell(sanitizeGraderId(grader))}: check failed`));
const errorLines = summary.errors.map((error) => `- ERROR: ${escapeCell(error)}`);
if (failureLines.length || errorLines.length) {
- lines.push(...failureLines, ...errorLines, "");
+ singleRunLines.push(...failureLines, ...errorLines, "");
+ }
+
+ if (!baselineSummary) {
+ return singleRunLines.join("\n");
}
- if (baselineSummary) {
- const comparison = compareSummaries(baselineSummary, summary);
+ const comparison = compareSummaries(baselineSummary, summary);
+ const skillVerdicts = summarizeSkillVerdicts(comparison);
+ const hasNoShip = skillVerdicts.some((s) => s.verdict === "๐ No ship") || summary.failedCount > 0;
+ const hasRerun = skillVerdicts.some((s) => s.verdict === "โ ๏ธ Re-run")
+ || summary.errorCount > 0
+ || summary.errors.length > 0
+ || baselineSummary.errors.length > 0;
+ const overallVerdict = hasNoShip ? "๐ No ship" : hasRerun ? "โ ๏ธ Re-run" : "โ
Ship";
+ const baseClean = (baselineSummary.failedCount + baselineSummary.errorCount) === 0 && baselineSummary.errors.length === 0;
+ const candClean = (summary.failedCount + summary.errorCount) === 0 && summary.errors.length === 0;
+ const baseOutcome = baseClean ? "passed" : "failed";
+ const candOutcome = overallVerdict === "โ
Ship"
+ ? "passed (โ
Ship)"
+ : overallVerdict === "๐ No ship"
+ ? "failed (๐ No ship)"
+ : `${candClean ? "passed" : "failed"} (โ ๏ธ Re-run)`;
+ const outcomeDelta = overallVerdict === "โ
Ship"
+ ? (baseOutcome === "failed" ? "fixed" : "unchanged")
+ : overallVerdict === "๐ No ship"
+ ? (baseOutcome === "passed" || skillVerdicts.some((s) => s.broke > 0) ? "regressed" : "๐ No ship")
+ : "โ ๏ธ Re-run";
+ const defaultModel = summary.models?.[0] ?? baselineSummary.models?.[0] ?? "default";
+
+ const actionableSkills = skillVerdicts.filter((s) => s.verdict !== "โ
Ship");
+ const passingChangedSkills = skillVerdicts.filter((s) => s.verdict === "โ
Ship" && (s.changed || skillVerdicts.length === 1));
+ const unchangedPassingSkills = skillVerdicts.filter((s) => s.verdict === "โ
Ship" && !s.changed && skillVerdicts.length > 1);
+
+ const lines = [
+ `## Evals: ${overallVerdict}`,
+ "",
+ "| Skill | Verdict | Why |",
+ "| --- | --- | --- |",
+ ...skillVerdicts.map((s) => {
+ const boldVerdict = s.verdict.replace(/^(โ
|๐|โ ๏ธ)\s+(.*)$/u, "$1 **$2**");
+ return `| **${escapeCell(s.skillName)}** | ${boldVerdict} | ${s.why} |`;
+ }),
+ "",
+ ];
+
+ for (const skill of actionableSkills) {
+ lines.push(...renderSkillDetailsBlock(skill, defaultModel));
+ }
+
+ if (actionableSkills.length > 0) {
+ lines.push("---", "");
+ }
+
+ for (const skill of passingChangedSkills) {
+ lines.push(...renderSkillDetailsBlock(skill, defaultModel));
+ }
+
+ if (unchangedPassingSkills.length > 0) {
+ const unchangedScenarios = unchangedPassingSkills.reduce((sum, s) => sum + s.candTotal, 0);
lines.push(
- "## Baseline vs. Candidate Comparison",
+ "",
+ `๐ Unchanged evals & run configuration โ ${unchangedPassingSkills.length} passing (${unchangedScenarios}/${unchangedScenarios} scenarios)
`,
"",
- ...(baselinePath ? [`Baseline: \`${escapeCell(baselinePath)}\` | Candidate: \`${escapeCell(path)}\``, ""] : []),
- "| Metric | Baseline | Candidate | Delta |",
+ "| Skill | Status | Scenarios (Before โ After) | Details |",
"| --- | --- | --- | --- |",
- `| Passed | ${baselineSummary.passedCount} | ${summary.passedCount} | ${formatSignedInt(comparison.deltas.passed)} |`,
- `| Failed | ${baselineSummary.failedCount + baselineSummary.errorCount} | ${summary.failedCount + summary.errorCount} | ${formatSignedInt(comparison.deltas.failed)} |`,
- `| Avg Score | ${formatScore(baselineSummary.averageScore)} | ${formatScore(summary.averageScore)} | ${formatScoreDelta(comparison.deltas.averageScore)} |`,
- `| Total Turns | ${baselineSummary.totalTurns ?? "โ"} | ${summary.totalTurns ?? "โ"} | ${formatSignedInt(comparison.deltas.totalTurns)} |`,
- `| Total Tokens | ${formatTokens(baselineSummary.tokenUsage)} | ${formatTokens(summary.tokenUsage)} | ${formatTokenDelta(comparison.deltas.totalTokens, comparison.deltas.totalTokensPct)} |`,
- ...(typeof baselineSummary.tokenUsage?.outputTokens === "number" && typeof summary.tokenUsage?.outputTokens === "number"
- ? [`| Output Tokens | ${baselineSummary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${summary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${formatTokenDelta(comparison.deltas.outputTokens, comparison.deltas.outputTokensPct)} |`]
- : []),
- `| Agent Wall Time | ${formatDuration(baselineSummary.totalWallTimeMs)} | ${formatDuration(summary.totalWallTimeMs)} | ${formatDurationDelta(comparison.deltas.totalWallTimeMs)} |`,
- `| Total Duration | ${formatDuration(baselineSummary.totalDurationMs)} | ${formatDuration(summary.totalDurationMs)} | ${formatDurationDelta(comparison.deltas.totalDurationMs)} |`,
+ ...unchangedPassingSkills.map((s) =>
+ `| \`${escapeCell(s.skillName)}\` | โ
Unchanged | ${s.basePassed}/${s.baseTotal} โ ${s.candPassed}/${s.candTotal} | All scenarios passing on \`${escapeCell(defaultModel)}\` |`,
+ ),
"",
- "| Case | Baseline | Candidate | Score Delta | Turn Delta | Token Delta | Wall Time Delta | Transition |",
- "| --- | --- | --- | --- | --- | --- | --- | --- |",
- ...comparison.cases.map((item) => {
- const baseCell = item.baseline ? `${item.baseline.status} (${formatScore(item.baseline.score)})` : "โ";
- const candCell = item.candidate ? `${item.candidate.status} (${formatScore(item.candidate.score)})` : "โ";
- return `| ${escapeCell(item.key)} | ${baseCell} | ${candCell} | ${formatScoreDelta(item.scoreDelta)} | ${formatSignedInt(item.turnsDelta)} | ${formatTokenDelta(item.tokenDelta, item.tokenDeltaPct)} | ${formatDurationDelta(item.wallTimeDeltaMs, item.wallTimeDeltaPct)} | ${item.transition} |`;
- }),
+ `Run config: ${comparison.cases.length} scenarios across ${skillVerdicts.length} ${skillVerdicts.length === 1 ? "eval" : "evals"} ยท ${escapeCell(defaultModel)}`,
"",
- "> **Note:** Single-run score, token, and duration deltas are observed signals for review rather than multi-trial statistical proof.",
+ " ",
"",
);
}
+ lines.push(
+ "",
+ `๐ Telemetry, token deltas & run evidence โ ${skillVerdicts.length} ${skillVerdicts.length === 1 ? "skill" : "skills"} evaluated (${comparison.cases.length} ${comparison.cases.length === 1 ? "scenario" : "scenarios"})
`,
+ "",
+ "## Baseline vs. Candidate Comparison",
+ "",
+ ...(baselinePath ? [`Baseline: \`${escapeCell(baselinePath)}\` | Candidate: \`${escapeCell(path)}\``, ""] : []),
+ "| Metric | Baseline | Candidate | Delta |",
+ "| --- | --- | --- | --- |",
+ `| Outcome | ${baseOutcome} | ${candOutcome} | ${outcomeDelta} |`,
+ `| Passed | ${baselineSummary.passedCount} | ${summary.passedCount} | ${formatSignedInt(comparison.deltas.passed)} |`,
+ `| Failed | ${baselineSummary.failedCount + baselineSummary.errorCount} | ${summary.failedCount + summary.errorCount} | ${formatSignedInt(comparison.deltas.failed)} |`,
+ `| Avg Score | ${formatScore(baselineSummary.averageScore)} | ${formatScore(summary.averageScore)} | ${formatScoreDelta(comparison.deltas.averageScore)} |`,
+ `| Total Turns | ${baselineSummary.totalTurns ?? "โ"} | ${summary.totalTurns ?? "โ"} | ${formatSignedInt(comparison.deltas.totalTurns)} |`,
+ `| Total Tokens | ${formatTokens(baselineSummary.tokenUsage)} | ${formatTokens(summary.tokenUsage)} | ${formatTokenDelta(comparison.deltas.totalTokens, comparison.deltas.totalTokensPct)} |`,
+ ...(typeof baselineSummary.tokenUsage?.outputTokens === "number" && typeof summary.tokenUsage?.outputTokens === "number"
+ ? [`| Output Tokens | ${baselineSummary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${summary.tokenUsage.outputTokens.toLocaleString("en-US")} | ${formatTokenDelta(comparison.deltas.outputTokens, comparison.deltas.outputTokensPct)} |`]
+ : []),
+ `| Agent Wall Time | ${formatDuration(baselineSummary.totalWallTimeMs)} | ${formatDuration(summary.totalWallTimeMs)} | ${formatDurationDelta(comparison.deltas.totalWallTimeMs)} |`,
+ `| Total Duration | ${formatDuration(baselineSummary.totalDurationMs)} | ${formatDuration(summary.totalDurationMs)} | ${formatDurationDelta(comparison.deltas.totalDurationMs)} |`,
+ "",
+ "| Case | Baseline | Candidate | Score Delta | Turn Delta | Token Delta | Wall Time Delta | Transition |",
+ "| --- | --- | --- | --- | --- | --- | --- | --- |",
+ ...comparison.cases.map((item) => {
+ const baseCell = item.baseline ? `${item.baseline.status} (${formatScore(item.baseline.score)})` : "โ";
+ const candCell = item.candidate ? `${item.candidate.status} (${formatScore(item.candidate.score)})` : "โ";
+ return `| ${escapeCell(item.key)} | ${baseCell} | ${candCell} | ${formatScoreDelta(item.scoreDelta)} | ${formatSignedInt(item.turnsDelta)} | ${formatTokenDelta(item.tokenDelta, item.tokenDeltaPct)} | ${formatDurationDelta(item.wallTimeDeltaMs, item.wallTimeDeltaPct)} | ${item.transition} |`;
+ }),
+ "",
+ "> **Note:** Single-run score, token, and duration deltas are observed signals for review rather than multi-trial statistical proof.",
+ "",
+ ...singleRunLines,
+ " ",
+ "",
+ );
+
return lines.join("\n");
}
diff --git a/.github/scripts/eval-report.test.mjs b/.github/scripts/eval-report.test.mjs
index 125ae00..3204138 100644
--- a/.github/scripts/eval-report.test.mjs
+++ b/.github/scripts/eval-report.test.mjs
@@ -16,6 +16,7 @@ import {
loadResults,
renderReport,
summarizeRun,
+ summarizeSkillVerdicts,
} from "./eval-report.mjs";
const root = fileURLToPath(new URL("../../", import.meta.url));
@@ -255,6 +256,10 @@ test("compareSummaries and renderReport surface fixed failures, regressions, and
baselineSummary: baseline,
baselinePath: "baseline/results.jsonl",
});
+ assert.match(markdown, /## Evals: โ
Ship/);
+ assert.match(markdown, /\| \*\*pr-description\*\* \| โ
\*\*Ship\*\* \| 1 scenario fixed \(1\/2 โ 2\/2 passing\) ยท -64\.0% output tokens \|/);
+ assert.match(markdown, /โ
pr-description<\/strong> โ 1 fixed \(1\/2 โ 2\/2 passing\)<\/summary>/);
+ assert.match(markdown, /๐ Telemetry, token deltas & run evidence<\/strong>/);
assert.match(markdown, /## Baseline vs\. Candidate Comparison/);
assert.match(markdown, /\| Total Turns \| 3 \| 3 \| 0 \|/);
assert.match(markdown, /\| Total Tokens \| 6,960 \(in: 5,570, out: 1,390\) \| 3,300 \(in: 2,800, out: 500\) \| -3,660 \(-52\.6%\) \|/);
@@ -263,6 +268,43 @@ test("compareSummaries and renderReport surface fixed failures, regressions, and
assert.match(markdown, /\| pr-description \/ pr-description-drafts-structured-pr-body \| BEHAVIORAL FAILURE \(50\.0%\) \| PASS \(100\.0%\) \| \+50\.0% \| 0 \| -2,410 \(-54\.8%\) \| -2\.4s \(-46\.2%\) \| Fixed \|/);
assert.match(markdown, /\| pr-description \/ pr-description-ignores-bug-and-review-question \| PASS \(100\.0%\) \| PASS \(100\.0%\) \| 0\.0% \| 0 \| -1,250 \(-48\.8%\) \| -1\.3s \(-41\.9%\) \| Reduced tokens at parity \|/);
assert.match(markdown, /Single-run score, token, and duration deltas are observed signals for review/);
+
+ // Multi-skill mixed verdict ordering: No ship and Re-run above ---, Ship and Unchanged/Telemetry below ---
+ const multiSkillBase = summarizeRun([
+ makeTrial({ runId: "ms-base", evalName: "pr-description", stimulus: "drafts-body", passed: false, gradeScore: 0.5, outputTokens: 900, inputTokens: 2000, totalTokens: 2900, details: [{ name: "program", passed: false, score: 0 }, { name: "completed", passed: true, score: 1 }] }),
+ makeTrial({ runId: "ms-base", evalName: "release-plugins", stimulus: "preserves-version", passed: true, gradeScore: 1, outputTokens: 400, inputTokens: 2000, totalTokens: 2400, details: [{ name: "skill-invocation", passed: true, score: 1 }] }),
+ makeTrial({ runId: "ms-base", evalName: "try-plugin-locally", stimulus: "prepares-session", passed: true, gradeScore: 1, outputTokens: 500, inputTokens: 2000, totalTokens: 2500 }),
+ makeTrial({ runId: "ms-base", evalName: "update-remote-pins", stimulus: "updates-pins", passed: true, gradeScore: 1, outputTokens: 300, inputTokens: 1500, totalTokens: 1800 }),
+ makeFooter("ms-base"),
+ ]);
+ const multiSkillCand = summarizeRun([
+ makeTrial({ runId: "ms-cand", evalName: "pr-description", stimulus: "drafts-body", passed: true, gradeScore: 1, outputTokens: 400, inputTokens: 2000, totalTokens: 2400, details: [{ name: "program", passed: true, score: 1 }, { name: "completed", passed: true, score: 1 }] }),
+ makeTrial({ runId: "ms-cand", evalName: "release-plugins", stimulus: "preserves-version", passed: false, gradeScore: 0, outputTokens: 390, inputTokens: 2000, totalTokens: 2390, details: [{ name: "skill-invocation", passed: false, score: 0 }] }),
+ makeTrial({ runId: "ms-cand", evalName: "try-plugin-locally", stimulus: "prepares-session", status: "error", passed: false, gradeScore: 0, outputTokens: 0, inputTokens: 0, totalTokens: 0 }),
+ makeTrial({ runId: "ms-cand", evalName: "update-remote-pins", stimulus: "updates-pins", passed: true, gradeScore: 1, outputTokens: 300, inputTokens: 1500, totalTokens: 1800 }),
+ makeFooter("ms-cand", true),
+ ]);
+ const multiSkillComparison = compareSummaries(multiSkillBase, multiSkillCand);
+ const verdicts = summarizeSkillVerdicts(multiSkillComparison);
+ assert.deepEqual(verdicts.map((v) => [v.skillName, v.verdict]), [
+ ["release-plugins", "๐ No ship"],
+ ["try-plugin-locally", "โ ๏ธ Re-run"],
+ ["pr-description", "โ
Ship"],
+ ["update-remote-pins", "โ
Ship"],
+ ]);
+ const multiSkillMarkdown = renderReport(multiSkillCand, "cand/results.jsonl", {
+ baselineSummary: multiSkillBase,
+ baselinePath: "base/results.jsonl",
+ });
+ assert.match(multiSkillMarkdown, /## Evals: ๐ No ship/);
+ const hrIndex = multiSkillMarkdown.indexOf("\n---\n");
+ assert.ok(hrIndex > 0, "expected --- separator between actionable and passing sections");
+ assert.ok(multiSkillMarkdown.indexOf("๐ release-plugins") < hrIndex);
+ assert.ok(multiSkillMarkdown.indexOf("โ ๏ธ try-plugin-locally") < hrIndex);
+ assert.ok(multiSkillMarkdown.indexOf("โ
pr-description") > hrIndex);
+ assert.ok(multiSkillMarkdown.indexOf("๐ Unchanged evals & run configuration") > hrIndex);
+ assert.ok(multiSkillMarkdown.indexOf("๐ Telemetry, token deltas & run evidence") > hrIndex);
+ assert.match(multiSkillMarkdown, /\| Outcome \| failed \| failed \(๐ No ship\) \|/);
});
prDescriptionTest("eval-report CLI compares the shared pr-description baseline and candidate fixtures without writing GITHUB_STEP_SUMMARY", async (t) => {
@@ -442,6 +484,19 @@ test("eval-report redacts raw grader evidence and error strings, requires comple
assert.match(mismatchedTrialsReport, /\| Total Tokens \| 1,100 \(in: 900, out: 200\) \| 400 \(in: 300, out: 100\) \| โ \|/);
assert.match(mismatchedTrialsReport, /\| Agent Wall Time \| 2\.2s \| 900ms \| โ \|/);
assert.match(mismatchedTrialsReport, /\| Total Duration \| 2\.2s \| 900ms \| โ \|/);
+ assert.doesNotMatch(mismatchedTrialsReport, /output tokens/);
+
+ // Footer execution error without trial failure still triggers Re-run and Outcome alignment
+ const footerErrCand = summarizeRun([
+ makeTrial({ runId: "f-cand", stimulus: "repeat-case", passed: true, gradeScore: 1, durationMs: 900, inputTokens: 300, outputTokens: 100, totalTokens: 400 }),
+ makeFooter("f-cand", true),
+ ]);
+ const footerErrReport = renderReport(footerErrCand, "cand/results.jsonl", {
+ baselineSummary: singleTrialCand,
+ baselinePath: "base/results.jsonl",
+ });
+ assert.match(footerErrReport, /## Evals: โ ๏ธ Re-run/);
+ assert.match(footerErrReport, /\| Outcome \| passed \| failed \(โ ๏ธ Re-run\) \| โ ๏ธ Re-run \|/);
const extraCaseBase = summarizeRun([
makeTrial({ runId: "e-base", stimulus: "repeat-case", passed: true, gradeScore: 1, durationMs: 900, inputTokens: 300, outputTokens: 100, totalTokens: 400 }),
diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md
index 0a54480..a512d63 100644
--- a/docs/checks-and-evals.md
+++ b/docs/checks-and-evals.md
@@ -308,15 +308,18 @@ matching `evals//skills//eval.yaml` changes:
3. **Post sticky PR comment (`report`):** run `github/evals-action` with
`command: report`, `report-id: evals`, and
`baseline-report-id: evals-baseline` in a separate `pull-requests: write`
- job to publish one sticky PR comment with the **Baseline vs. Candidate
- comparison** table (Outcome, Passed/Failed, Score delta, Duration delta, and
- Token `in/out` + `%` delta) alongside per-eval results.
+ job to publish one sticky PR comment with a top-level **Ship / No-ship /
+ Re-run** verdict summary table and a collapsible **Telemetry, token deltas &
+ run evidence** `` section (`Outcome`, `Passed/Failed`, `Score
+ delta`, `Turns delta`, `Duration delta`, and `Token in/out + %` delta).
> **Action version note:** `baseline-report-id` (`evals-baseline`) is provided
> by [github/evals-action#34](https://github.com/github/evals-action/pull/34)
-> (verified commit `3c551330898c1467b8ddf9ef7c7f6e6c6af489d6`). Pin
-> `github/evals-action@3c551330898c1467b8ddf9ef7c7f6e6c6af489d6` until `#34`
-> is merged and published to `github/evals-action@v0`.
+> (`3c551330898c1467b8ddf9ef7c7f6e6c6af489d6`), and the top-level **Ship /
+> No-ship** verdict table layout is added in
+> [github/evals-action#35](https://github.com/github/evals-action/pull/35). Pin
+> `github/evals-action` to that commit until published to
+> `github/evals-action@v0`.
```yaml
name: Skill eval comparison