Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 156 additions & 18 deletions .github/scripts/eval-report.mjs

Large diffs are not rendered by default.

248 changes: 92 additions & 156 deletions .github/scripts/eval-report.test.mjs

Large diffs are not rendered by default.

130 changes: 130 additions & 0 deletions .github/scripts/pr-description.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
import test from "node:test";
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { createDefaultGraderRegistry } from "@microsoft/vally";
import { parse } from "yaml";

const root = fileURLToPath(new URL("../../", import.meta.url));
const prDescriptionTest = existsSync(resolve(root, "plugins/pr-description")) ? test : test.skip;

prDescriptionTest("pr-description eval stages catalog-local skills and enforces non-collision", async () => {
const evalDir = resolve(root, "evals/pr-description/skills/pr-description");
const spec = parse(readFileSync(resolve(evalDir, "eval.yaml"), "utf8"));
assert.equal(spec.scoring.threshold, 1);
const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
assert.equal(spec.agent_environment, "marketplace");
const skills = project.environments.marketplace.skills;
for (const configuredSkill of skills) {
assert.ok(existsSync(resolve(evalDir, configuredSkill)), `configured cohort skill path is missing: ${configuredSkill}`);
}
const staged = new Set(skills.map((skill) => resolve(evalDir, skill)));
const catalog = JSON.parse(readFileSync(resolve(root, "marketplace.json"), "utf8"));
for (const plugin of catalog.plugins.filter((plugin) => typeof plugin.source === "string")) {
const directory = resolve(root, plugin.source, "skills");
if (!existsSync(directory)) continue;
for (const skill of readdirSync(directory)) {
const source = resolve(directory, skill);
if (existsSync(resolve(source, "SKILL.md"))) {
assert.ok(staged.has(source), `Add catalog skill ${plugin.name}/${skill} to the native marketplace environment.`);
}
}
}

const grader = createDefaultGraderRegistry().get("skill-invocation");
const positive = spec.stimuli.find((item) => item.tags?.control === "positive");
const negative = spec.stimuli.find((item) => item.tags?.control === "negative");
const posConfig = positive.graders.find((item) => item.type === "skill-invocation").config;
const negConfig = negative.graders.find((item) => item.type === "skill-invocation").config;
const activation = (name) => ({ type: "skill_activation", data: { name } });

const exactMatch = await grader.grade({
stimulus: positive, config: posConfig,
trajectory: { output: "ok", events: [activation("pr-description")] },
});
assert.equal(exactMatch.passed, true);

if (posConfig.disallowed?.length) {
const neighborCollision = await grader.grade({
stimulus: positive, config: posConfig,
trajectory: { output: "ok", events: [activation("pr-description"), activation(posConfig.disallowed[0])] },
});
assert.equal(neighborCollision.passed, false, "activating a configured neighbor must fail");
}

const negativeTriggered = await grader.grade({
stimulus: negative, config: negConfig,
trajectory: { output: "ok", events: [activation("pr-description")] },
});
assert.equal(negativeTriggered.passed, false, "activating pr-description on the negative case must fail");
});

prDescriptionTest("pr-description program grader enforces ordered headings, factual changes, and unshipped backlog exclusion", async (t) => {
const gradingDir = resolve(root, ".github/scripts", `.pr-grader-test-${randomUUID()}`);
mkdirSync(gradingDir, { recursive: true });
t.after(() => rmSync(gradingDir, { recursive: true, force: true }));
copyFileSync(resolve(root, "evals/pr-description/skills/pr-description/grader.mjs"), resolve(gradingDir, "grader.mjs"));
copyFileSync(resolve(root, "evals/_shared/graders/common.mjs"), resolve(gradingDir, "common.mjs"));
const { evaluate: evaluatePrDescription } = await import(pathToFileURL(resolve(gradingDir, "grader.mjs")));
const { score } = await import(pathToFileURL(resolve(gradingDir, "common.mjs")));
const validOutput = [
"## Summary",
"Validate `--timeout` as a positive integer in `src/cli.mjs` and add unit tests. Retry backoff (#42) remains open and out of scope.",
"",
"## Changes",
"- Added positive integer validation for `--timeout` in `src/cli.mjs`.",
"- Added unit tests for `--timeout 0` and `--timeout -5` in `test/cli.test.mjs`.",
"",
"## Verification",
"- `npm test` passed (14 tests).",
"",
"## Risks / Rollback",
"- Low risk; revert the commit if needed.",
].join("\n");
assert.equal(score("pr-description-contract", evaluatePrDescription({ output: validOutput, events: [] })).passed, true);
for (const passingLine of [
"- `npm test` (14 tests): passed.",
"- `npm test` \u2014 14 tests, all passing.",
"- Ran `npm test`; 14/14 tests passed.",
"- `npm test`: 14 tests passed, 0 failed, 0 errors.",
"- `npm test` passed (14 tests, no errors).",
"- `npm test`: 14 passed, # fail 0, 0 failures.",
]) {
const output = validOutput.replace("- `npm test` passed (14 tests).", passingLine);
assert.equal(score("pr-description-contract", evaluatePrDescription({ output, events: [] })).passed, true, passingLine);
}
for (const negatedLine of [
"- `npm test` (14 tests).",
"- `npm test` did not pass (14 tests).",
"- `npm test` (14 tests): not passing.",
"- `npm test`: 14 tests; 12 passed, 2 failures.",
"- `npm test` failed (14 tests).",
]) {
const output = validOutput.replace("- `npm test` passed (14 tests).", negatedLine);
const result = score("pr-description-contract", evaluatePrDescription({ output, events: [] }));
assert.equal(result.passed, false, negatedLine);
assert.match(result.evidence, /Recorded the supplied npm test verification result/);
}
for (const leakedChanges of [
"- Added unit tests for `--timeout 0` and `--timeout -5` in `test/cli.test.mjs`.\n- Implemented exponential retry backoff (#42).",
"- Rejects `--timeout` values that are not positive integers.\n- Adds exponential retry backoff (#42).",
]) {
const output = validOutput.replace(
"- Added unit tests for `--timeout 0` and `--timeout -5` in `test/cli.test.mjs`.", leakedChanges);
const result = score("pr-description-contract", evaluatePrDescription({ output, events: [] }));
assert.equal(result.passed, false);
assert.match(result.evidence, /Excluded unshipped retry backoff/);
}
const missingRisks = validOutput.split("## Risks / Rollback")[0];
assert.equal(score("pr-description-contract", evaluatePrDescription({ output: missingRisks, events: [] })).passed, false);
for (const prohibitedTool of ["bash", "functions.bash", "functions.powershell", "mcp:edit", "tools/write_file"]) {
const result = score("pr-description-contract", evaluatePrDescription({
output: validOutput,
events: [{ type: "tool_call", data: { toolCallId: "1", toolName: prohibitedTool, arguments: { command: "git status" } } }],
}));
assert.equal(result.passed, false, `must reject prohibited tool ${prohibitedTool}`);
assert.match(result.evidence, /Did not run shell commands or create a pull request/);
}
});
8 changes: 7 additions & 1 deletion .github/workflows/evals.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
name: Evals
# Runs the smoke evals with real Copilot sessions, which cost AI credits.
# Start by explicit manual dispatch only; pull requests never start paid runs.
# This full smoke-suite workflow starts by explicit manual dispatch only.
# Same-repository PRs that change plugins/<plugin>/skills/<skill>/ with a matching
# evals/<plugin>/skills/<skill>/eval.yaml run .github/workflows/skill-eval-comparison.yml
# (using github/evals-action@3c551330898c1467b8ddf9ef7c7f6e6c6af489d6 from
# github/evals-action#34 with report-id: evals-baseline and
# baseline-report-id: evals-baseline to update one sticky PR comment automatically);
# see docs/checks-and-evals.md#automatic-pr-skill-comparison.
on:
workflow_dispatch:
permissions:
Expand Down
141 changes: 141 additions & 0 deletions .github/workflows/skill-eval-comparison.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
name: Skill eval comparison
on:
pull_request:
Comment thread
jongio marked this conversation as resolved.
Resolved
paths:
- plugins/*/skills/**
- evals/*/skills/**
- .github/workflows/skill-eval-comparison.yml
Comment thread
jongio marked this conversation as resolved.
Resolved
workflow_dispatch:

permissions:
contents: read

jobs:
select:
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
skill_dirs: ${{ steps.changed.outputs.skill_dirs }}
eval_paths: ${{ steps.changed.outputs.eval_paths }}
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
fetch-depth: 0
persist-credentials: false
- id: changed
env:
EVENT_NAME: ${{ github.event_name }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
dirs=("plugins/pr-description/skills/pr-description")
else
mapfile -t dirs < <(
git diff --name-only "$BASE_SHA" HEAD -- 'plugins/*/skills/*' \
| awk -F/ 'NF>=4 {print "plugins/"$2"/"$3"/"$4}' \
| sort -u
)
fi
skill_dirs=()
eval_paths=()
for dir in "${dirs[@]}"; do
[ -n "$dir" ] || continue
eval_path="evals/${dir#plugins/}/eval.yaml"
if git cat-file -e "${BASE_SHA:-HEAD}:$dir/SKILL.md" 2>/dev/null && [ -f "$dir/SKILL.md" ] && [ -f "$eval_path" ]; then
skill_dirs+=("$dir")
eval_paths+=("$eval_path")
fi
done
{
echo "skill_dirs<<EOF"
printf '%s\n' "${skill_dirs[@]}"
echo "EOF"
echo "eval_paths<<EOF"
printf '%s\n' "${eval_paths[@]}"
echo "EOF"
} >> "$GITHUB_OUTPUT"

run-baseline:
needs: select
if: needs.select.outputs.eval_paths != ''
runs-on: ubuntu-latest
Comment thread
jongio marked this conversation as resolved.
Resolved
timeout-minutes: 15
permissions:
contents: read
copilot-requests: write
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
fetch-depth: 0
persist-credentials: false
- name: Restore baseline skill sources
env:
EVENT_NAME: ${{ github.event_name }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
SKILL_DIRS: ${{ needs.select.outputs.skill_dirs }}
run: |
while IFS= read -r dir; do
[ -n "$dir" ] || continue
skill_name="${dir##*/}"
fixture_path="evals/_shared/fixtures/${skill_name}/SKILL.verbose-fixture.md"
if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -f "$fixture_path" ]; then
cp "$fixture_path" "$dir/SKILL.md"
elif [ -n "$BASE_SHA" ]; then
rm -rf -- "$dir"
git checkout "$BASE_SHA" -- "$dir"
fi
done <<< "$SKILL_DIRS"
- uses: github/evals-action@3c551330898c1467b8ddf9ef7c7f6e6c6af489d6
continue-on-error: true
env:
VALLY_TELEMETRY_OPTOUT: '1'
with:
command: run
working-directory: .
eval-paths: ${{ needs.select.outputs.eval_paths }}
report-id: evals-baseline
copilot-token: ${{ github.token }}
max-retries: 0

run:
needs: select
if: needs.select.outputs.eval_paths != ''
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
copilot-requests: write
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
persist-credentials: false
- uses: github/evals-action@3c551330898c1467b8ddf9ef7c7f6e6c6af489d6
env:
VALLY_TELEMETRY_OPTOUT: '1'
with:
command: run
working-directory: .
eval-paths: ${{ needs.select.outputs.eval_paths }}
report-id: evals
copilot-token: ${{ github.token }}
max-retries: 0

report:
needs: [select, run-baseline, run]
if: always() && needs.select.outputs.eval_paths != '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
actions: read
contents: read
pull-requests: write
concurrency:
group: evals-${{ github.workflow_ref }}
cancel-in-progress: false
steps:
- uses: github/evals-action@3c551330898c1467b8ddf9ef7c7f6e6c6af489d6
with:
command: report
report-id: evals
baseline-report-id: evals-baseline
26 changes: 23 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,25 @@ request.
Local examples use a dedicated Copilot Requests token with public-repository
access for the synthetic fixtures, not a broad GitHub CLI login credential.
Then `npm run eval` runs real Copilot sessions and costs AI credits.
[Evals](.github/workflows/evals.yml) runs only when manually started; pull
requests never trigger paid evals.
[Evals](.github/workflows/evals.yml) runs the full smoke suite only when
manually started (`workflow_dispatch`).

**Baseline vs. Candidate skill comparison:** locally, run
`npm run eval:compare -- --baseline <base-results.jsonl> --candidate <cand-results.jsonl>`
to compare two saved Vally runs (verdicts, score deltas, model, skills invoked,
turns, total and output token deltas, agent wall time deltas, total duration,
and per-case transitions) without calling a model. On same-repository pull
requests (`head.repo.full_name == github.repository`, excluding fork PRs),
[Skill eval comparison](.github/workflows/skill-eval-comparison.yml) uses
`github/evals-action` (`report-id: evals-baseline` and
`baseline-report-id: evals-baseline`, introduced in
[github/evals-action#34](https://github.com/github/evals-action/pull/34) at
commit `3c551330898c1467b8ddf9ef7c7f6e6c6af489d6` and included in `@v0` once
released) so any changed skill under
`plugins/<plugin>/skills/<skill>/` with a matching
`evals/<plugin>/skills/<skill>/eval.yaml` automatically runs Baseline vs.
Candidate and updates one sticky PR comparison comment; see
[automatic PR skill comparison](docs/checks-and-evals.md#automatic-pr-skill-comparison).

Repository evals use **Vally's default local backend**, without an advanced
project backend setting. This keeps [.vally.yaml](.vally.yaml) within the GitHub
Expand All @@ -211,7 +228,10 @@ review eval setup, tools, network access and credential exposure before approval
`npm run eval -- --suite full` selects every retained case, including the
independent reviewer agent, and still spends AI credits. See
[execution and authentication](docs/checks-and-evals.md#execution-and-authentication)
for the compatibility and isolation limits. Paid CI remains manual-only.
for the compatibility and isolation limits. The smoke-suite workflow remains
manual-only; teams wanting a manual approval gate on PR skill comparisons can
optionally add a protected `environment:` to the comparison workflow's execution
jobs.

**Standard models:** `npm run eval:models` uses the native
[model experiment](evals/standard-models.experiment.yaml) to test every retained
Expand Down
Loading
Loading