Skip to content
12 changes: 7 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -254,11 +254,13 @@ Artifacts uploaded by older Action versions require explicit IDs/digests.

### Baseline vs. Candidate PR comparison

To include a **Baseline vs. Candidate comparison** table (Outcome, Passed/Failed,
Score delta, Turns delta, Duration delta, and Token `in/out` + `%` delta) in the sticky PR
comment and job summary, run the baseline arm with a distinct `report-id`
(such as `evals-baseline`), run the candidate arm with `report-id: evals`, and
pass `baseline-report-id: evals-baseline` to the `report` step:
To include a top-level **Ship / No-ship / Re-run verdict summary table** (`βœ… Ship`,
`πŸ›‘ No ship`, `⚠️ Re-run`) and a collapsible **Telemetry, token deltas & run
evidence** `<details>` section (`Baseline vs. Candidate comparison`, `Quality &
token summary`, and evidence links) in the sticky PR comment, run the baseline
arm with a distinct `report-id` (such as `evals-baseline`), run the candidate
arm with `report-id: evals`, and pass `baseline-report-id: evals-baseline` to
the `report` step:

```yaml
report:
Expand Down
2 changes: 1 addition & 1 deletion dist/index.cjs

Large diffs are not rendered by default.

47 changes: 33 additions & 14 deletions src/reporter.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import { parseDocument } from 'yaml';
import { commentMarker, orderedBody, selectComment, type Comment, type CommentIdentity, type ReportOrder } from './comments.js';
import { object } from './policy.js';
import { escapeMarkdown, hasQualityDetails, renderComparisonSection, renderQualitySection, type Report } from './reporting.js';
import {
escapeMarkdown, hasQualityDetails, renderComparisonSection, renderQualitySection, renderVerdictSummaryTable,
type Report,
} from './reporting.js';
import { ReportingError, type GitHubClient } from './github.js';

export const commentConcurrencyGroup = 'evals-${{ github.workflow_ref }}';
Expand Down Expand Up @@ -171,7 +174,8 @@ export function renderComment(
const status = evaluation?.outcome ?? lint?.outcome ?? report.outcome;
const rows = evaluation?.groups ?? [];
const suite = evaluation?.suite;
const base = `# Evals: ${status}\n\n` +
const maxEvalRows = baseline ? 25 : 100;
const detailsBody =
(suite ? `**Suite:** ${escapeMarkdown(suite)}\n\n` : evaluation ? '**Selection:** Native eval paths or tags\n\n' : '') +
(lint ? `**Lint:** ${lint.outcome}${lint.outcome === 'validated' ? ' (no model execution)' : ` - ${escapeMarkdown(lint.code)}`}` +
((companion?.artifactId ?? artifactId) !== undefined ? ` - [Evidence](${run}/artifacts/${companion?.artifactId ?? artifactId})` : '') + '\n\n' : '') +
Expand All @@ -181,21 +185,36 @@ export function renderComment(
(['error', 'failed', 'cancelled'].includes(evaluation.outcome) ? `Diagnostic: ${escapeMarkdown(evaluation.code)}.\n\n` : '') +
(evaluation.outcome === 'not-run' ? '**Not an evaluation pass.**\n\n' : '') +
(rows.length ? '| Eval | Trials | Result |\n| --- | ---: | --- |\n' +
rows.slice(0, 100).map((group) =>
rows.slice(0, maxEvalRows).map((group) =>
`| ${escapeMarkdown(group.name ?? group.identity.slice(0, 12))} | ${group.trials} | ${group.trials ? group.passed ? 'passed' : 'failed' : 'not-run'} |`,
).join('\n') + '\n\n' +
(rows.length > 100 ? `${rows.length - 100} further groups are in the structured artifact.\n\n` : '') : '') : '');
if (!evaluation) return base;
let output = base;
(rows.length > maxEvalRows ? `${rows.length - maxEvalRows} further groups are in the structured artifact.\n\n` : '') : '') : '');
if (!evaluation) return `# Evals: ${status}\n\n${detailsBody}`;
if (!baseline) {
let output = `# Evals: ${status}\n\n${detailsBody}`;
if (rows.length && hasQualityDetails(rows)) {
const maxQualityBytes = Math.min(16_000, Math.max(0, 57_500 - Buffer.byteLength(output, 'utf8')));
const quality = renderQualitySection('### Quality & token summary', rows, 25, maxQualityBytes);
if (quality) output = `${output}${quality}\n\n`;
}
return output;
}

const { overallVerdict, markdown: verdictTable } = renderVerdictSummaryTable(baseline.report, evaluation, 25);
const header = `# Evals: ${overallVerdict} (${status})\n\n` +
(verdictTable ? `${verdictTable}\n\n` : '') +
(['error', 'failed', 'cancelled'].includes(evaluation.outcome) ? `Diagnostic: ${escapeMarkdown(evaluation.code)}.\n\n` : '') +
(evaluation.outcome === 'not-run' ? '**Not an evaluation pass.**\n\n' : '');

let telemetry = detailsBody;
if (rows.length && hasQualityDetails(rows)) {
const maxQualityBytes = Math.min(16_000, Math.max(0, 57_500 - Buffer.byteLength(output, 'utf8')));
const maxQualityBytes = Math.min(16_000, Math.max(0, 56_000 - Buffer.byteLength(header + telemetry, 'utf8')));
const quality = renderQualitySection('### Quality & token summary', rows, 25, maxQualityBytes);
if (quality) output = `${output}${quality}\n\n`;
if (quality) telemetry = `${telemetry}${quality}\n\n`;
}
if (baseline) {
const maxComparisonBytes = Math.min(16_000, Math.max(0, 57_500 - Buffer.byteLength(output, 'utf8')));
const comparison = renderComparisonSection('### Baseline vs. Candidate comparison', baseline.report, evaluation, 25, maxComparisonBytes);
if (comparison) output = `${output}${comparison}\n\n`;
}
return output;
const maxComparisonBytes = Math.min(16_000, Math.max(0, 56_000 - Buffer.byteLength(header + telemetry, 'utf8')));
const comparison = renderComparisonSection('### Baseline vs. Candidate comparison', baseline.report, evaluation, 25, maxComparisonBytes, overallVerdict);
if (comparison) telemetry = `${telemetry}${comparison}\n\n`;

return `${header}<details>\n<summary>πŸ“Š <strong>Telemetry, token deltas &amp; run evidence</strong></summary>\n\n${telemetry.trim()}\n\n</details>\n\n`;
Comment thread
jongio marked this conversation as resolved.
Resolved
}
176 changes: 169 additions & 7 deletions src/reporting.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,15 +208,177 @@ function formatGroupCell(group?: ReportGroup): string {
return group.overallScore !== undefined ? `${verdict} (${group.overallScore.toFixed(2)})` : verdict;
}

export interface GroupVerdictEntry {
key: string;
name: string;
baseGroup: ReportGroup | undefined;
candGroup: ReportGroup | undefined;
verdict: 'βœ… Ship' | 'πŸ›‘ No ship' | '⚠️ Re-run';
transition: string;
changed: boolean;
why: string;
}

export function classifyGroupVerdict(baseline: ReportGroup | undefined, candidate: ReportGroup | undefined): {
verdict: 'βœ… Ship' | 'πŸ›‘ No ship' | '⚠️ Re-run';
transition: string;
changed: boolean;
why: string;
} {
const transition = classifyGroupTransition(baseline, candidate);
if (!candidate || candidate.trials === 0) {
return {
verdict: '⚠️ Re-run',
transition,
changed: true,
why: !candidate ? 'Eval removed from candidate run' : 'No trials executed (not-run)',
};
}
if (baseline && baseline.trials === 0 && candidate.passed) {
return {
verdict: '⚠️ Re-run',
transition,
changed: true,
why: 'Baseline had no trials executed (not-run)',
};
}
const ship = candidate.passed && transition !== 'regressed';
const verdict = ship ? 'βœ… Ship' : 'πŸ›‘ No ship';
const changed = transition !== 'unchanged';
const parts: string[] = [];
if (baseline && baseline.trials === 0 && !candidate.passed) {
parts.push('Failed in candidate (baseline not-run)');
} else if (!candidate.passed) {
if (transition === 'regressed' && baseline?.passed) parts.push('Regressed to failing');
else if (transition === 'regressed') parts.push('Still failing (score regressed)');
else if (transition === 'improved-score') parts.push('Still failing (score improved)');
else if (transition === 'added') parts.push('New eval failing');
else parts.push('Still failing');
} else if (transition === 'fixed') {
parts.push('Fixed failing eval');
} else if (transition === 'regressed') {
parts.push('Score regressed');
} else if (transition === 'improved-score') {
parts.push('Improved score');
} else if (transition === 'reduced-tokens') {
parts.push('Reduced token usage at parity');
} else if (transition === 'increased-tokens') {
parts.push('Increased token usage at parity');
} else if (transition === 'added') {
parts.push('New eval passing');
} else {
parts.push(`Passing (${candidate.stimuliRun}/${candidate.stimuliTotal} scenarios)`);
}
Comment thread
jongio marked this conversation as resolved.
Resolved
if (baseline?.overallScore !== undefined && candidate.overallScore !== undefined) {
parts.push(`score ${baseline.overallScore.toFixed(2)} β†’ ${candidate.overallScore.toFixed(2)} (${formatScoreDelta(baseline.overallScore, candidate.overallScore)})`);
} else if (candidate.overallScore !== undefined) {
parts.push(`score ${candidate.overallScore.toFixed(2)}`);
}
const sameTrials = baseline !== undefined && baseline.trials === candidate.trials;
if (sameTrials && baseline?.tokens?.output !== undefined && candidate.tokens?.output !== undefined && baseline.tokens.output > 0) {
const outDelta = candidate.tokens.output - baseline.tokens.output;
if (outDelta !== 0) {
const sign = outDelta > 0 ? '+' : '';
const pct = ((outDelta / baseline.tokens.output) * 100).toFixed(1);
parts.push(`${sign}${pct}% output tokens (${sign}${outDelta})`);
}
}
if (sameTrials && baseline?.durationMs !== undefined && candidate.durationMs !== undefined && baseline.durationMs > 0) {
const durDelta = candidate.durationMs - baseline.durationMs;
if (durDelta !== 0) {
const sign = durDelta > 0 ? '+' : '';
const pct = ((durDelta / baseline.durationMs) * 100).toFixed(1);
parts.push(`${sign}${pct}% duration`);
}
}
return { verdict, transition, changed, why: parts.join(' Β· ') };
}

export function renderVerdictSummaryTable(
baseline: Report | undefined, candidate: Report, maxGroups = 25, maxBytes = 16_000,
): { overallVerdict: 'βœ… Ship' | 'πŸ›‘ No ship' | '⚠️ Re-run'; entries: GroupVerdictEntry[]; markdown: string } {
const baseGroups = baseline?.groups ?? [];
const candGroups = candidate.groups ?? [];
const baseMap = new Map(baseGroups.map((group) => [group.identity, group]));
const candMap = new Map(candGroups.map((group) => [group.identity, group]));
const keys = [...new Set([...baseMap.keys(), ...candMap.keys()])];
const baseIncomplete = baseline !== undefined && ['error', 'not-run', 'cancelled'].includes(baseline.outcome);
const candIncomplete = ['error', 'not-run', 'cancelled'].includes(candidate.outcome);
const severityOrder: Record<'πŸ›‘ No ship' | '⚠️ Re-run' | 'βœ… Ship', number> = {
'πŸ›‘ No ship': 0,
'⚠️ Re-run': 1,
'βœ… Ship': 2,
};
const entries: GroupVerdictEntry[] = keys
.map((key) => {
const baseGroup = baseMap.get(key);
const candGroup = candMap.get(key);
const name = escapeMarkdown((candGroup ?? baseGroup)!.name ?? (candGroup ?? baseGroup)!.identity.slice(0, 12));
let { verdict, transition, changed, why } = classifyGroupVerdict(baseGroup, candGroup);
if (verdict !== 'πŸ›‘ No ship' && (baseIncomplete || candIncomplete)) {
verdict = '⚠️ Re-run';
changed = true;
if (baseIncomplete && !baseGroup) why = 'Baseline did not complete';
else if (candIncomplete) why = 'Candidate run did not complete';
}
return { key, name, baseGroup, candGroup, verdict, transition, changed, why };
})
.sort((a, b) => severityOrder[a.verdict] - severityOrder[b.verdict]);

if (entries.length === 0) {
const overallVerdict = candidate.outcome === 'failed'
? 'πŸ›‘ No ship'
: candidate.outcome === 'passed' && !baseIncomplete
? 'βœ… Ship'
: '⚠️ Re-run';
return { overallVerdict, entries, markdown: '' };
}
let hasNoShip = candidate.outcome === 'failed';
let hasRerun = candIncomplete || baseIncomplete;
for (const entry of entries) {
if (entry.verdict === 'πŸ›‘ No ship') hasNoShip = true;
if (entry.verdict === '⚠️ Re-run') hasRerun = true;
}
const overallVerdict = hasNoShip ? 'πŸ›‘ No ship' : hasRerun ? '⚠️ Re-run' : 'βœ… Ship';
Comment thread
jongio marked this conversation as resolved.
Resolved
if (maxBytes <= 0) return { overallVerdict, entries, markdown: '' };
const header = '| Skill | Verdict | Why |\n| --- | --- | --- |\n';
const footer = '\n\nFurther groups are available in the structured artifact.';
const footerBytes = Buffer.byteLength(footer, 'utf8');
let usedBytes = Buffer.byteLength(header, 'utf8');
const rows: string[] = [];
const limit = Math.min(entries.length, maxGroups);
for (let index = 0; index < limit; index++) {
const entry = entries[index]!;
const boldVerdict = entry.verdict.replace(/^(βœ…|πŸ›‘|⚠️)\s+(.*)$/u, '$1 **$2**');
const row = `| **${entry.name}** | ${boldVerdict} | ${entry.why} |`;
const rowBytes = Buffer.byteLength((rows.length ? '\n' : '') + row, 'utf8');
const needsFooter = index + 1 < entries.length;
if (usedBytes + rowBytes + (needsFooter ? footerBytes : 0) > maxBytes) break;
rows.push(row);
usedBytes += rowBytes;
}
const markdown = rows.length === 0
? ''
: header + rows.join('\n') + (rows.length < entries.length ? footer : '');
return { overallVerdict, entries, markdown };
}

export function renderComparisonSection(
heading: string, baseline: Report, candidate: Report, maxGroups = 25, maxBytes = 16_000,
heading: string, baseline: Report, candidate: Report, maxGroups = 25, maxBytes = 16_000, overallVerdict?: string,
): string {
if (maxBytes <= 0) return '';
const outcomeTransition = baseline.outcome === 'failed' && candidate.outcome === 'passed'
? 'fixed'
: baseline.outcome === 'passed' && candidate.outcome === 'failed'
? 'regressed'
: baseline.outcome === candidate.outcome ? 'unchanged' : `${baseline.outcome} -> ${candidate.outcome}`;
const hasGroupRegression = (baseline.groups ?? []).some((baseGroup) => {
const candGroup = (candidate.groups ?? []).find((g) => g.identity === baseGroup.identity);
return classifyGroupTransition(baseGroup, candGroup) === 'regressed';
});
const outcomeTransition = (baseline.outcome === 'passed' && candidate.outcome === 'failed') || hasGroupRegression
? 'regressed'
: overallVerdict === '⚠️ Re-run'
? '⚠️ Re-run'
: baseline.outcome === 'failed' && candidate.outcome === 'passed' && overallVerdict !== 'πŸ›‘ No ship'
? 'fixed'
: baseline.outcome === candidate.outcome ? 'unchanged' : `${baseline.outcome} -> ${candidate.outcome}`;
const candidateOutcomeCell = overallVerdict ? `${candidate.outcome} (${overallVerdict})` : candidate.outcome;
const hasBothGroups = baseline.groups !== undefined && candidate.groups !== undefined;
const baseGroups = baseline.groups ?? [];
const candGroups = candidate.groups ?? [];
Expand All @@ -234,7 +396,7 @@ export function renderComparisonSection(
const summaryTable =
`${heading}\n\n` +
'| Metric | Baseline | Candidate | Delta |\n| --- | --- | --- | --- |\n' +
`| Outcome | ${baseline.outcome} | ${candidate.outcome} | ${outcomeTransition} |\n` +
`| Outcome | ${baseline.outcome} | ${candidateOutcomeCell} | ${outcomeTransition} |\n` +
`| Passed | ${baseline.passed ?? '-'} | ${candidate.passed ?? '-'} | ${formatSignedCount(baseline.passed, candidate.passed)} |\n` +
`| Failed | ${baseline.failed ?? '-'} | ${candidate.failed ?? '-'} | ${formatSignedCount(baseline.failed, candidate.failed)} |\n` +
`| Turns | ${baseline.turns ?? '-'} | ${candidate.turns ?? '-'} | ${workloadsMatch ? formatSignedCount(baseline.turns, candidate.turns) : '-'} |\n` +
Expand Down
52 changes: 52 additions & 0 deletions tests/reporter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,62 @@ describe('one sticky PR report for direct Action calls', () => {
report: baselineReport, artifactId: 125,
})).toBe('created');
const body = String(test.writes()[0]?.[2]?.body);
expect(body).toContain('# Evals: βœ… Ship (passed)');
expect(body).toContain('| Skill | Verdict | Why |');
expect(body).toContain('| **pr-description** | βœ… **Ship** | Fixed failing eval Β· score 0.94 β†’ 1.00 (+0.06) Β· -31.5% output tokens (-456) Β· -11.4% duration |');
expect(body).toContain('<summary>πŸ“Š <strong>Telemetry, token deltas &amp; run evidence</strong></summary>');
expect(body).toContain('**Baseline:** failed - 2 trials - [Evidence](https://github.com/o/r/actions/runs/1000/artifacts/125)');
expect(body).toContain('### Baseline vs. Candidate comparison');
expect(body).toContain('| Outcome | failed | passed (βœ… Ship) | fixed |');
expect(body).toContain('| pr-description | failed (0.94) | passed (1.00) | +0.06 | -2 | -4401ms (-11.4%) | -191 tokens (-0.3%; out: -456 / -31.5%) | fixed |');
Comment thread
jongio marked this conversation as resolved.
Resolved

// Mixed verdicts (No ship, Re-run, still-failing token deltas) and 100-group comment byte budget
const mixedBase: Report = {
...report, reportId: 'smoke-baseline', suite: 'smoke', outcome: 'failed',
executed: 2, passed: 1, failed: 1, turns: 4, durationMs: 20_000,
tokens: { input: 40_000, output: 1_000, total: 41_000 },
groups: [
{ identity: '1'.repeat(64), name: 'still-failing', passed: false, trials: 1, stimuliRun: 1, stimuliTotal: 1, skipped: 0, overallScore: 0.5, threshold: 1, turns: 2, durationMs: 10_000, tokens: { input: 20_000, output: 600, total: 20_600 } },
{ identity: '2'.repeat(64), name: 'skipped-cand', passed: true, trials: 1, stimuliRun: 1, stimuliTotal: 1, skipped: 0, overallScore: 1, threshold: 1, turns: 2, durationMs: 10_000, tokens: { input: 20_000, output: 400, total: 20_400 } },
],
};
const mixedCand: Report = {
...report, suite: 'smoke', outcome: 'failed',
executed: 1, passed: 0, failed: 1, turns: 2, durationMs: 9_000,
tokens: { input: 20_000, output: 500, total: 20_500 },
groups: [
{ identity: '1'.repeat(64), name: 'still-failing', passed: false, trials: 1, stimuliRun: 1, stimuliTotal: 1, skipped: 0, overallScore: 0.5, threshold: 1, turns: 2, durationMs: 9_000, tokens: { input: 20_000, output: 500, total: 20_500 } },
{ identity: '2'.repeat(64), name: 'skipped-cand', passed: true, trials: 0, stimuliRun: 0, stimuliTotal: 1, skipped: 1 },
],
};
const mixedTest = fixture();
expect(await postReport(mixedTest.client, context, mixedCand, 123, undefined, { report: mixedBase, artifactId: 125 })).toBe('created');
const mixedBody = String(mixedTest.writes()[0]?.[2]?.body);
expect(mixedBody).toContain('# Evals: πŸ›‘ No ship (failed)');
expect(mixedBody).toContain('| **still-failing** | πŸ›‘ **No ship** | Still failing Β· score 0.50 β†’ 0.50 (0.00) Β· -16.7% output tokens (-100) Β· -10.0% duration |');
expect(mixedBody).toContain('| **skipped-cand** | ⚠️ **Re-run** | No trials executed (not-run) |');

const heavyGroups = Array.from({ length: 100 }, (_, index) => ({
identity: index.toString(16).padStart(64, '0'),
name: `${'&'.repeat(90)}-${index}`,
model: 'claude-haiku-5.5',
passed: true,
trials: 1,
stimuliRun: 1,
stimuliTotal: 1,
skipped: 0,
overallScore: 1,
threshold: 1,
turns: 2,
durationMs: 50,
tokens: { input: 20, output: 30, total: 50 },
}));
const heavyBase: Report = { ...report, reportId: 'smoke-baseline', suite: 'smoke', outcome: 'passed', executed: 100, passed: 100, failed: 0, turns: 200, durationMs: 5000, tokens: { input: 2000, output: 3000, total: 5000 }, groups: heavyGroups };
const heavyCand: Report = { ...report, suite: 'smoke', outcome: 'passed', executed: 100, passed: 100, failed: 0, turns: 200, durationMs: 5000, tokens: { input: 2000, output: 3000, total: 5000 }, groups: heavyGroups };
const heavyTest = fixture();
expect(await postReport(heavyTest.client, context, heavyCand, 123, undefined, { report: heavyBase, artifactId: 125 })).toBe('created');
expect(Buffer.byteLength(String(heavyTest.writes()[0]?.[2]?.body), 'utf8')).toBeLessThanOrEqual(60_000);

for (const badBaseline of [
{ report: { ...baselineReport, reportId: 'smoke' }, artifactId: 125 },
{ report: { ...baselineReport, command: 'lint' }, artifactId: 125 },
Expand Down
Loading