From 47bc8c3fe7ce838d0ca27a8e9b84c54a5377d4f5 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Fri, 30 Oct 2026 10:00:23 -0700 Subject: [PATCH 1/3] Remove the GitHub Copilot Evals plugin from the catalog github/evals isn't public yet and uses a different license, so this MIT template shouldn't point at it. Removed the remote eval entry with npm run remove-plugin, and dropped the docs, comments, and tests that named github/evals. The eval file name guidance now cites Vally's defaults. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vally.yaml | 4 ++-- README.md | 17 ++++------------- docs/customization-types.md | 4 ++-- docs/evals.md | 11 ++++++----- docs/remote-plugins.md | 30 ------------------------------ marketplace.json | 13 ------------- test/adoption.test.mjs | 5 ++--- test/catalog.test.mjs | 11 ++--------- 8 files changed, 18 insertions(+), 77 deletions(-) diff --git a/.vally.yaml b/.vally.yaml index 33de6d7..e265e11 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -10,8 +10,8 @@ paths: - plugins/json-configuration/skills evals: - evals - # eval.yaml is this repository's convention; *.eval.yaml is the upstream - # github/evals convention. Both work. + # eval.yaml is Vally's default. *.eval.yaml lets one folder hold several + # specs, such as release.eval.yaml. Both work. evalFilenames: - eval.yaml - "*.eval.yaml" diff --git a/README.md b/README.md index 248b0ee..da8321b 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,6 @@ Two terms come up a lot: | [code-review](plugins/code-review/) | "Use adversarial-review to review this diff without changing it" | A skill that delegates to a read-only custom agent | | [json-configuration](plugins/json-configuration/) | 'Use json-configuration to validate {"port":8443} and look up /port' | A skill backed by a bundled local MCP server | - -The catalog also has an optional reference to the GitHub Copilot Evals plugin, -which lives in another repository. See [Remote plugins](docs/remote-plugins.md). - [Examples](docs/examples.md) explains what each example shows and which files it owns. @@ -104,12 +100,7 @@ You need Node.js 22.14 or newer. 4. Add your first plugin with `npm run new-plugin -- `. See [Add a plugin](#add-a-plugin). `npm run verify` passes even with no plugins at all. - -5. Decide whether to keep the remote `eval` plugin in the catalog. See - [Remote plugins](docs/remote-plugins.md). Remove it with - `npm run remove-plugin -- eval`. - -6. Replace what's meant for you before you ship: +5. Replace what's meant for you before you ship: - Your own copyright line in [LICENSE](LICENSE), beside the template's. That one file covers the whole repository, including every plugin. See [Licensing](#licensing). @@ -123,15 +114,15 @@ You need Node.js 22.14 or newer. - Your maintenance rules in [.github/copilot-instructions.md](.github/copilot-instructions.md) and [AGENTS.md](AGENTS.md). -7. Run `npm run verify`. It fails on anything inconsistent and warns about +6. Run `npm run verify`. It fails on anything inconsistent and warns about leftovers, such as the template's name or introduction, a `LICENSE` that doesn't name you yet, scaffold text you haven't replaced, or a mention of a plugin you removed. -8. Push, then turn on branch protection or a ruleset for your default branch. +7. Push, then turn on branch protection or a ruleset for your default branch. Rulesets are available for public repositories on every plan. Behavioral evals stay off in CI until you turn them on; see [Continuous integration](#continuous-integration). -9. [Test installation](docs/local-development.md#test-installation-without-publishing) +8. [Test installation](docs/local-development.md#test-installation-without-publishing) locally, then from your GitHub repository, before you share it. For agent-assisted setup and maintenance, see [AGENTS.md](AGENTS.md). diff --git a/docs/customization-types.md b/docs/customization-types.md index d15eb8d..9988c29 100644 --- a/docs/customization-types.md +++ b/docs/customization-types.md @@ -31,8 +31,8 @@ create or turn on an automation. See VS Code's `lsp.json` is checked against the fields in the plugin reference: `lspServers`, a launch command, and `fileExtensions`. The reference's component -table doesn't list extensions; this repository and the published GitHub Copilot -Evals plugin both ship them in `com.github.copilot/extensions/`. +table doesn't list extensions; this repository ships them in +`com.github.copilot/extensions/`. ## Where each type works diff --git a/docs/evals.md b/docs/evals.md index 2fe5800..aa60564 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -84,9 +84,10 @@ and check your billing dashboard after a run to learn what your suite costs. ## Write an eval 1. Create `evals//eval.yaml` with realistic tasks, expected outcomes, - and at least one negative or non-trigger case. The upstream - [github/evals](https://github.com/github/evals) convention, - `.eval.yaml`, works too; `evalFilenames` in `.vally.yaml` lists both. + and at least one negative or non-trigger case. `eval.yaml` is + [Vally's default file name](https://microsoft.github.io/vally/reference/eval-spec/). + `.eval.yaml` works too, so one folder can hold several specs; + `evalFilenames` in `.vally.yaml` lists both. Keep private production data out of fixtures and expected answers out of agent-visible inputs. Put grader programs and answer keys in `grading_environment`. @@ -218,8 +219,8 @@ them. Test them by hand in the hosts you support. ## Suites Suites are conventions, not fixed names. This repository's `smoke` suite -selects cases tagged `suite: smoke`, and `full` runs every spec. Upstream -github/evals examples filter on `lane: ci` instead. A `tags.suite` value must +selects cases tagged `suite: smoke`, and `full` runs every spec. You can +filter on any tag, such as `lane: ci`. A `tags.suite` value must match a suite filter in `.vally.yaml`, and every asset needs a case in `smoke` because CI runs it by default. diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index 417279f..7776d36 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -11,36 +11,6 @@ run upstream code, and the local evals don't cover remote plugins: `evals/` is for the plugins under `plugins/`, and the coverage check rejects asset IDs that aren't local. Evaluate an upstream plugin in its own repository. - -## The GitHub Copilot Evals plugin - -[marketplace.json](../marketplace.json) includes an `eval` entry that -points at the published GitHub Copilot Evals plugin in -[github/evals](https://github.com/github/evals). It mirrors the stable entry in -upstream's own catalog: the `copilot-evals-v0.20.3` release, pinned to the full -SHA of its built commit. - -After adding this marketplace, install it explicitly with -`copilot plugin install eval@MARKETPLACE`, where `MARKETPLACE` is the -`name` in `marketplace.json`. Or skip this catalog and install it from -upstream, which follows upstream's reviewed stable releases: - -```sh -copilot plugin marketplace add github/evals -copilot plugin install eval@github-copilot-evals-dist -``` - -Pick one route. Don't install the same plugin from both marketplaces. - -The pinned release is the built plugin. It provides the native `/eval` command -(`/eval setup`, `create`, `review`, `improve`, `run`, and `compare`) in the CLI -and Copilot App, a canvas, and six model-invocable skills. Running an eval -needs a compatible runner and its own approval, and it spends AI credits. -This repository's eval tooling works without it. - -This catalog entry doesn't copy upstream code. If you don't want it, remove the -entry with `npm run remove-plugin -- eval`. - ## Add a remote plugin diff --git a/marketplace.json b/marketplace.json index 60fe4a5..7e85a9b 100644 --- a/marketplace.json +++ b/marketplace.json @@ -37,19 +37,6 @@ "source": "./plugins/json-configuration", "version": "0.1.6", "description": "Validate JSON and retrieve configuration values with a local MCP server." - }, - { - "name": "eval", - "source": { - "source": "github", - "repo": "github/evals", - "ref": "copilot-evals-v0.20.3", - "sha": "acd4b5029b0791fbe1c4b58e28fa94c580841d69", - "path": "packages/plugin" - }, - "version": "0.20.3", - "description": "Configure native projects with /eval setup, create and save evals with /eval create, save distinct candidates with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running follow separate host approval policies.", - "homepage": "https://github.com/github/evals#installed-skills" } ] } diff --git a/test/adoption.test.mjs b/test/adoption.test.mjs index 7baf7b2..ca2c64a 100644 --- a/test/adoption.test.mjs +++ b/test/adoption.test.mjs @@ -124,7 +124,7 @@ test("a team can rename the marketplace, drop examples and the remote entry, and assert.match(coverage, new RegExp(`agent:${fixture}-tools/${fixture}-tools-specialist`)); assert.match(coverage, new RegExp(`mcp:${fixture}-tools/${fixture}-tools/echo`)); for (const name of dropped) assert.doesNotMatch(coverage, new RegExp(`:${name}/`)); - const params = await ok(directory, "scripts/evals.mjs", "action-params"); + const params = JSON.stringify(parse(await readFile(join(directory, ".vally.yaml"), "utf8")).params ?? {}); for (const name of dropped) assert.doesNotMatch(params, new RegExp(name)); // Scaffolding refuses names that would collide or aren't valid plugin names. @@ -183,11 +183,10 @@ test("an empty or remote-only marketplace passes verify, and new plugins scaffol assert.ok((await json(join(directory, "marketplace.json"))).plugins.every((entry) => typeof entry.source === "string")); const empty = await verify(directory, { tests: false }); if (!ownPlugins.length) assert.match(empty.coverage, /nothing to cover/); - // The setup steps about examples and the remote plugin go with them, and the list stays numbered in order. + // The setup steps about examples go with them, and the list stays numbered in order. const setup = (await readFile(join(directory, "README.md"), "utf8")).match(/## Make it yours\n[\s\S]*?\n## /)?.[0]; if (setup) { if (present.length) assert.doesNotMatch(setup, /remove-example/); - assert.doesNotMatch(setup, /plugin:eval\b|remove-plugin -- eval\b/); const numbers = [...setup.matchAll(/^(\d+)\. /gm)].map((match) => Number(match[1])); assert.deepEqual(numbers, numbers.map((_, index) => index + 1)); } diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs index 7f0209c..460e874 100644 --- a/test/catalog.test.mjs +++ b/test/catalog.test.mjs @@ -9,14 +9,7 @@ import { discoverAssets, checkCoverage, root } from "../scripts/validate-evals.m test("the catalog separates local plugins from pinned, opt-in remote references", async () => { const { catalog, localPlugins, remotePlugins } = await readCatalog(root); assert.equal(localPlugins.length + remotePlugins.length, catalog.plugins.length); - const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals"); - if (evals) { - // The catalog name must match upstream's plugin.json name, not its marketplace name. - assert.equal(evals.name, "eval"); - assert.equal(evals.source.path, "packages/plugin"); - assert.deepEqual(remoteWarnings(evals), []); - assert.equal(evals.version, evals.source.ref.replace(/^copilot-evals-v/, "")); - } + for (const remote of remotePlugins) assert.doesNotThrow(() => validateRemoteSource(remote.source, remote.name)); const local = { name: "dup", source: "./plugins/dup" }; const sha = "a".repeat(40); assert.throws(() => partitionCatalog({ ...catalog, plugins: [local, local] }), /Duplicate/); @@ -37,7 +30,7 @@ test("an empty catalog is valid, and catalog errors are plain sentences", () => test("remote references must pin a full sha, and errors name the entry", () => { const sha = "a".repeat(40); for (const source of [ - { source: "github", repo: "github/evals", path: "packages/plugin", sha }, + { source: "github", repo: "example/plugins", path: "packages/plugin", sha }, { source: "github", repo: "example/plugin", ref: "v1.0.0", sha }, { source: "github", repo: "example/plugin", sha }, ]) assert.doesNotThrow(() => validateRemoteSource(source, "fixture")); From c8ea1d47a6f305336b8d5e55fc708fa39848e6dd Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Fri, 30 Oct 2026 10:00:29 -0700 Subject: [PATCH 2/3] Run CI evals with the pinned Vally CLI only github/evals-action isn't public, so the evals workflow now has one job that runs the same launcher as local runs, with the Vally CLI pinned in package.json. Removed the EVALS_RUNNER switch, the action-params command and its gitignored params files, and the upload-transcripts warning, which only applied to the Action. Docs now point at Vally's public docs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/evals.yml | 92 ++------------------- .gitignore | 4 - AGENTS.md | 14 ++-- README.md | 13 ++- docs/ci.md | 102 ++++++++---------------- docs/security.md | 11 ++- evals/customization-authoring/eval.yaml | 4 +- scripts/catalog.mjs | 10 +-- scripts/evals.mjs | 40 +--------- scripts/validate.mjs | 14 +--- test/eval-setup.test.mjs | 78 +++--------------- test/marketplace.test.mjs | 2 - 12 files changed, 76 insertions(+), 308 deletions(-) diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 8c3f7b6..b4b702c 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -1,11 +1,10 @@ name: Behavioral evals # Off by default, because evals run real Copilot sessions and spend AI credits. # Pull requests run evals only after you set the repository variable -# ENABLE_EVALS to true. Manual runs work either way. The repository variable -# EVALS_RUNNER picks the job: unset or "action" uses github/evals-action; -# "npm" runs the pinned Vally CLI from package.json. Any other value fails. Both eval -# jobs use the "evals" environment, so you can require a reviewer to approve -# each run before it gets a token. See docs/ci.md. +# ENABLE_EVALS to true. Manual runs work either way. The job runs the pinned +# Vally CLI from package.json through the same launcher contributors use +# locally. It uses the "evals" environment, so you can require a reviewer to +# approve each run before it gets a token. See docs/ci.md. on: pull_request: types: [opened, synchronize, reopened, ready_for_review] @@ -28,91 +27,14 @@ on: permissions: contents: read jobs: - # Fails fast on a mistyped EVALS_RUNNER instead of silently picking a runner. - check-runner: - if: vars.EVALS_RUNNER != '' && vars.EVALS_RUNNER != 'action' && vars.EVALS_RUNNER != 'npm' - runs-on: ubuntu-latest - permissions: {} - steps: - - run: | - echo "::error::EVALS_RUNNER is '$EVALS_RUNNER'. Set it to action, npm, or nothing. See docs/ci.md." - exit 1 - env: - EVALS_RUNNER: ${{ vars.EVALS_RUNNER }} - - # Runner "action". A separate job, because a job downloads every action it - # names before it starts, even ones in skipped steps. evals: # Manual runs always run. Pull requests run only when ENABLE_EVALS is true, # the branch is in this repository (never a fork), and the PR isn't a draft. if: >- - (vars.EVALS_RUNNER == '' || vars.EVALS_RUNNER == 'action') && (github.event_name == 'workflow_dispatch' || - (vars.ENABLE_EVALS == 'true' && - github.event.pull_request.head.repo.full_name == github.repository && - !github.event.pull_request.draft)) - runs-on: ubuntu-latest - # Add required reviewers to this environment to approve each run before it starts. - environment: evals - timeout-minutes: 30 - env: - VALLY_TELEMETRY_OPTOUT: '1' - permissions: - contents: read - copilot-requests: write - pull-requests: write - concurrency: - group: evals-${{ github.workflow_ref }} - cancel-in-progress: false - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: 24 - cache: npm - # The Action installs only Vally. Graders, eval MCP hosts, and the pinned - # full Copilot runtime for custom-agent delegation come from npm ci. - - run: npm ci --no-audit --no-fund - # The Action can't pass --param, so write gitignored eval-level params files - # with this runner's absolute paths from .vally.yaml params. - - run: node scripts/evals.mjs action-params - # Fail fast if the runtime bridge the Action forwards can't start the pinned - # runtime, instead of reporting confusing eval failures later. It prints - # the pinned package version and the version that actually runs. - - name: Check the runtime bridge - run: node "$COPILOT_CLI_PATH" --version - env: - COPILOT_CLI_PATH: ${{ github.workspace }}/scripts/eval-runtime.js - - uses: github/evals-action@83fa12332c3e44ad899f347eaf153a0d9b6432dc # v0.1.0 - env: - # Vally's SDK executor launches this bridge, which runs the pinned full runtime. - COPILOT_CLI_PATH: ${{ github.workspace }}/scripts/eval-runtime.js - with: - working-directory: . - suite: ${{ inputs.suite || 'smoke' }} - vally-version: '0.19.0' - copilot-token: ${{ github.token }} - # Workaround: the Action forwards only an allowlist of variables to Vally, - # and credential-env is its only way to pass another one, during the run - # phase only. COPILOT_CLI_PATH is a non-secret path, not a credential. - # Naming anything here turns off automatic token injection, so - # copilot-token is set explicitly above. - # VALLY_TELEMETRY_OPTOUT turns off Vally's telemetry during the run phase. - credential-env: | - COPILOT_CLI_PATH - VALLY_TELEMETRY_OPTOUT - workers: 1 - max-retries: 1 - upload-transcripts: false - - # Runner "npm": the same launcher contributors use locally, with the job token. - evals-npm: - if: >- - vars.EVALS_RUNNER == 'npm' && (github.event_name == 'workflow_dispatch' || + github.event_name == 'workflow_dispatch' || (vars.ENABLE_EVALS == 'true' && github.event.pull_request.head.repo.full_name == github.repository && - !github.event.pull_request.draft)) + !github.event.pull_request.draft) runs-on: ubuntu-latest # Add required reviewers to this environment to approve each run before it starts. environment: evals @@ -137,7 +59,7 @@ jobs: - run: npm ci --no-audit --no-fund - name: Check the runtime bridge run: node scripts/eval-runtime.js --version - # --require-pass fails the job on a failed verdict, like the Action does. + # --require-pass fails the job on a failed verdict. # The launcher uses one worker and one bounded retry. - name: Run evals run: node scripts/evals.mjs eval --suite "$SUITE" --require-pass diff --git a/.gitignore b/.gitignore index 099f667..f368ced 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,3 @@ vally-results/ .vally/ # Isolated Copilot homes from local install checks. .copilot-home/ -# Written by `node scripts/evals.mjs action-params` in CI. To commit a params file -# you wrote yourself, add it with `git add -f`; action-params won't overwrite it. -evals/**/eval.params.yaml -evals/**/*.eval.params.yaml diff --git a/AGENTS.md b/AGENTS.md index 9465fdb..7da55e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,12 +72,12 @@ place; don't duplicate them here. - `npm run eval:smoke`, `npm run eval`, and the evals workflow spend AI credits. Don't run them unless the user asks, and don't turn on `ENABLE_EVALS` for anyone. Static checks don't establish behavioral quality. -- When upgrading Vally or Copilot, update the pins, the lockfile, and the - workflow's `vally-version` together, and verify real custom-agent delegation - and MCP behavior, not only schema validity. When you touch the workflows, - refresh every action's commit SHA (with its version comment), keep the - evals workflow on a Node major that `validate.yml` tests, and keep both eval - runner jobs (`action` and `npm`) in step. +- When upgrading Vally or Copilot, update the pins in `package.json` and the + lockfile together, and verify real custom-agent delegation and MCP behavior, + not only schema validity. CI runs evals through the same launcher as local + runs. When you touch the workflows, refresh every action's commit SHA (with + its version comment), and keep the evals workflow on a Node major that + `validate.yml` tests. - Keep remote plugins opt-in. The repository settings overlay must never enable a remote plugin; that's a deliberate security rule, so developers install remote plugins themselves. A remote entry's `name` must match the upstream @@ -105,7 +105,7 @@ Consult the relevant contract before changing package or catalog fields: - [Creating a team marketplace](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/plugins-marketplace). - [CLI plugin and catalog reference](https://docs.github.com/en/copilot/reference/copilot-cli-reference/cli-plugin-reference). - [Vally eval schema](https://microsoft.github.io/vally/reference/eval-spec/) and - [Evals Action](https://github.com/github/evals-action). + [Vally CI guide](https://microsoft.github.io/vally/get-started/add-to-ci/). The README's [references](README.md#references) cover skills, hooks, VS Code, and this repository's guides. \ No newline at end of file diff --git a/README.md b/README.md index da8321b..d21c65b 100644 --- a/README.md +++ b/README.md @@ -250,11 +250,9 @@ marketplace consistent. [evals.yml](.github/workflows/evals.yml) runs behavioral evals. It's off for pull requests until you set the repository variable `ENABLE_EVALS` to `true`, and you can always run it by hand. Each run can wait for a reviewer's approval -through the `evals` environment. By default it uses -[`github/evals-action`](https://github.com/github/evals-action). Set the -repository variable `EVALS_RUNNER` to `npm` to run -the same evals with the pinned Vally CLI. On a personal account, usage is -billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers +through the `evals` environment. It runs the same launcher you use locally, +with the pinned Vally CLI. On a personal account, usage is billed to your own +Copilot plan. [Continuous integration](docs/ci.md) covers setup, personal accounts, billing, cost, and security. ## Repository layout @@ -345,5 +343,6 @@ to everyone who's contributed since. - [Hooks reference](https://docs.github.com/en/copilot/reference/hooks-reference) - [VS Code agent plugins](https://code.visualstudio.com/docs/agent-customization/agent-plugins) - [Agent Skills specification](https://agentskills.io/specification) -- [Vally](https://microsoft.github.io/vally/) and the - [Evals Action](https://github.com/github/evals-action) +- [Vally](https://microsoft.github.io/vally/), its + [eval schema](https://microsoft.github.io/vally/reference/eval-spec/), and + [CI guide](https://microsoft.github.io/vally/get-started/add-to-ci/) diff --git a/docs/ci.md b/docs/ci.md index 4b6a814..90370f2 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -39,8 +39,6 @@ organization: - `validate.yml` works as it is, and it's all you need to keep the marketplace consistent. It never calls a model. -- To run behavioral evals in CI, set the repository variable `EVALS_RUNNER` to - `npm`. - Copilot usage from the job's token is [billed to your own Copilot plan](https://docs.github.com/en/copilot/concepts/agent-capabilities/agent-powered-workflows-and-operations/copilot-cli-in-github-actions). Each plan includes a monthly allowance of AI credits; see @@ -68,10 +66,7 @@ turn it on. ### Turn it on 1. Check the [requirements](#requirements) below. -2. Pick a runner (see the next section). If you use the npm runner, create the - repository variable `EVALS_RUNNER` with the value `npm`. Any value other - than `action`, `npm`, or nothing fails the workflow instead of guessing. -3. Decide who approves runs. Both eval jobs use an environment named `evals`, +2. Decide who approves runs. The eval job uses an environment named `evals`, which GitHub creates the first time a job uses it. To make a person approve each run before it gets a token, add required reviewers to it in **Settings** > **Environments** > **evals**. On GitHub Free, Pro, and Team @@ -79,10 +74,10 @@ turn it on. repositories need GitHub Enterprise. See [deployment protection rules](https://docs.github.com/en/actions/reference/workflows-and-actions/deployments-and-environments#required-reviewers), and [Security](#security) for what this does and doesn't stop. -4. Try a manual run first: **Actions** > **Behavioral evals** > **Run +3. Try a manual run first: **Actions** > **Behavioral evals** > **Run workflow**, and choose `smoke`. Manual runs work whether or not evals are turned on for pull requests. -5. To run the `smoke` suite on pull requests, create the repository variable +4. To run the `smoke` suite on pull requests, create the repository variable `ENABLE_EVALS` with the value `true`. Repository variables live in **Settings** > **Secrets and variables** > **Actions** > **Variables**. @@ -95,66 +90,34 @@ Pull request runs start only when all of these hold: - It touches plugins, evals, schemas, scripts, `.vally.yaml`, the package files, or the workflow itself. -Runs don't cancel each other, because the Evals Action requires that, so each -push to an open pull request queues one more run. Keep work-in-progress pull +Runs don't cancel each other, so a paid run never stops halfway. Each push to +an open pull request queues one more run, so keep work-in-progress pull requests as drafts. -### Pick a runner - -| | `action` (default) | `npm` | -|---|---|---| -| Set `EVALS_RUNNER` to | nothing, or `action` | `npm` | -| Runs | [`github/evals-action`](https://github.com/github/evals-action) v0.1.0 | `node scripts/evals.mjs eval --suite --require-pass` | -| Who can use it | Any repository | Any repository | -| Report | Structured report and one sticky pull request comment | The job summary | -| Job token permissions | `contents: read`, `copilot-requests: write`, `pull-requests: write` | `contents: read`, `copilot-requests: write` | - -The two runners are separate jobs, because a job downloads every action it -names before it starts, even ones in skipped steps. With `EVALS_RUNNER=npm`, the -Action is never downloaded. - -Both fail the job on a failed verdict, an incomplete run, or an infrastructure -error, unlike the report-only default of `npm run eval:report` locally. Both use -one worker and one bounded retry, and both start by printing the pinned Copilot -CLI version and the version that actually runs. - -#### The Evals Action - -In its default mode the Action lints the project, runs the selected suite if -lint passes, uploads a structured report, and updates one sticky pull request -comment. It installs only Vally, pinned to the same version as `package.json`. -Its lint isn't `--strict` and doesn't lint skill directories, so -`npm run verify` stays the authoritative gate. Before it runs, the job: - -1. Sets up Node.js 24 and runs `npm ci`, so graders, eval MCP hosts, and the - pinned full Copilot runtime are available. -2. Runs `node scripts/evals.mjs action-params`. The Action can't pass - `--param`, so this writes gitignored eval-level params files - (`eval.params.yaml`, or `.eval.params.yaml` beside a - `.eval.yaml`) with the runner's absolute paths. Vally merges them over - the relative `.vally.yaml` defaults. A params file you wrote yourself has no - generated header, so `action-params` stops instead of overwriting it. -3. Checks that the runtime bridge starts the pinned runtime. -4. Sets `COPILOT_CLI_PATH` to the runtime bridge and names it in - `credential-env`. That's a workaround: the Action forwards only an allowlist - of variables to Vally, and `credential-env` is its only way to pass another - one, during the run phase. The value is a non-secret path. Because - `credential-env` is set, the workflow passes `copilot-token` explicitly. - -`upload-transcripts` stays `false` because raw transcripts can contain prompts, -model output, and code. Set it to `true` temporarily when you need to debug a -run, and review who can read workflow artifacts. `npm run verify` warns while -it's `true`, so it doesn't stay on by accident. See the Action's README for -separate lint, run, and report jobs if you need narrower permissions. - -#### The npm runner - -The npm runner uses the same launcher you use locally, with the job's -`GITHUB_TOKEN` passed to the Copilot runtime, as +### What the job does + +The job uses the same launcher you use locally, with the pinned +[Vally CLI](https://microsoft.github.io/vally/) from `package.json` and the +job's `GITHUB_TOKEN` passed to the Copilot runtime, as [Vally's CI guide](https://microsoft.github.io/vally/get-started/add-to-ci/) -describes. It writes the report to the job summary. Its results include -transcripts, so it uploads them as an artifact only when you set the -repository variable `EVALS_UPLOAD_RESULTS` to `true`. +describes. The job: + +1. Sets up Node.js 24 and runs `npm ci`, so Vally, graders, eval MCP hosts, and + the pinned full Copilot runtime are available. +2. Checks that the runtime bridge starts the pinned runtime. It prints the + pinned Copilot CLI version and the version that actually runs. +3. Runs `node scripts/evals.mjs eval --suite --require-pass`, with one + worker and one bounded retry. +4. Runs `node scripts/eval-report.mjs` and adds the report to the job summary. + +The job fails on a failed verdict, an incomplete run, or an infrastructure +error, unlike the report-only default of `npm run eval:report` locally. Its +token has `contents: read` and `copilot-requests: write`. + +Results include transcripts, which can contain prompts, model output, and code. +The job uploads them as an artifact only when you set the repository variable +`EVALS_UPLOAD_RESULTS` to `true`. Set it temporarily when you need to debug a +run, review who can read workflow artifacts, and unset it when you're done. ## Requirements @@ -170,11 +133,10 @@ repository variable `EVALS_UPLOAD_RESULTS` to `true`. - **`copilot-requests: write`.** The workflow grants this job permission so the token can make Copilot requests. See [using Copilot CLI in GitHub Actions with GITHUB_TOKEN](https://docs.github.com/en/copilot/how-tos/copilot-cli/use-copilot-cli-in-actions). -- **Runners.** The Evals Action supports Linux x64 GitHub-hosted runners only. - Both jobs use `ubuntu-latest`. +- **Runners.** The job uses `ubuntu-latest`. Other runners aren't verified. - **Actions policy.** If your organization restricts which actions can run, allow the actions the workflow pins: `actions/checkout`, `actions/setup-node`, - `actions/upload-artifact`, and, for the default runner, `github/evals-action`. + and `actions/upload-artifact`. ## Cost @@ -192,7 +154,7 @@ After your first manual run, check the dashboard to learn what a run costs. Same-repository pull requests run their own versions of plugin hooks, MCP servers, graders, and scripts with the eval job's token, which can make billed -Copilot requests (and, for the Action runner, write pull request comments). +Copilot requests. Merge review doesn't protect this: the code runs when the pull request is opened or updated, before anyone reviews it. Fork pull requests never get these runs. @@ -220,7 +182,7 @@ Before you turn on `ENABLE_EVALS`: - Every action is pinned to a full commit SHA, and checkout doesn't persist credentials. Keep it that way. -[Security and data handling](security.md#data-flow) lists what each runner +[Security and data handling](security.md#data-flow) lists what the job writes, who can read it, and for how long. ## Turn evals off diff --git a/docs/security.md b/docs/security.md index 6878e44..0908d2f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -3,7 +3,7 @@ This page is for whoever approves this marketplace for a team: what runs, when, with what access, and where data goes. [SECURITY.md](../SECURITY.md) covers reporting a vulnerability. [Continuous integration](ci.md#security) covers the -eval workflows in detail. +eval workflow in detail. None of this is a sandbox. Hooks, MCP servers, extensions, graders, and scripts run as ordinary processes with the permissions of the account that runs them. @@ -30,15 +30,14 @@ plugins. |---|---|---| | Prompts, files the agent reads, tool arguments and results, hook messages, MCP results, and canvas notes the agent reads | Copilot and the model behind it | Set by your Copilot plan and policies, not by this template. See the [GitHub Copilot Trust Center](https://copilot.github.trust.page/). | | Local eval results in `vally-results/`: results, a report, JUnit, and session evidence with transcripts | Your disk | Gitignored. Delete them when you're done. | -| npm runner in CI: Vally's log and the report in the job summary | The workflow run | Anyone who can read the repository's Actions runs. Logs follow the repository's [retention setting](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization) (90 days by default). | -| npm runner results artifact, which includes transcripts | A workflow artifact | Only when `EVALS_UPLOAD_RESULTS` is `true`. Kept for 7 days. | -| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. This template doesn't document the Action's internals. | -| Vally telemetry | Vally's telemetry service, unless turned off | `VALLY_TELEMETRY_OPTOUT=1` is set by `validate.yml`, the local launcher, and both eval jobs. The Action job also forwards it to Vally's run phase; whether the Action's lint and report phases honor it isn't verified. | +| The eval job in CI: Vally's log and the report in the job summary | The workflow run | Anyone who can read the repository's Actions runs. Logs follow the repository's [retention setting](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization) (90 days by default). | +| The eval job's results artifact, which includes transcripts | A workflow artifact | Only when `EVALS_UPLOAD_RESULTS` is `true`. Kept for 7 days. | +| Vally telemetry | Vally's telemetry service, unless turned off | `VALLY_TELEMETRY_OPTOUT=1` is set by `validate.yml`, the local launcher, and the eval job. | | Copilot CLI telemetry | GitHub | Set by Copilot's own settings and policies, not by this template. | Grader evidence can quote what the model said, and transcripts hold prompts, model output, and file contents. Keep every fixture, prompt, and expected -answer synthetic, and treat reports, comments, logs, and artifacts as possibly +answer synthetic, and treat reports, logs, and artifacts as possibly sensitive. Data a plugin keeps for itself, such as a canvas's state or an MCP server's diff --git a/evals/customization-authoring/eval.yaml b/evals/customization-authoring/eval.yaml index 17a1584..cf3d412 100644 --- a/evals/customization-authoring/eval.yaml +++ b/evals/customization-authoring/eval.yaml @@ -40,8 +40,8 @@ stimuli: - type: program config: program: node - # The Evals Action passes only allowlisted environment variables, so the - # pinned Vally CLI path arrives as an argument from a .vally.yaml param. + # The pinned Vally CLI path arrives as an argument from a .vally.yaml + # param, which the launcher passes as an absolute path. args: [.grading/authoring.mjs, '${MARKETPLACE_VALLY_CLI}'] timeout: 60s - name: does-not-author-for-factual-question diff --git a/scripts/catalog.mjs b/scripts/catalog.mjs index fe3a4e3..32f1c57 100644 --- a/scripts/catalog.mjs +++ b/scripts/catalog.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { readFile, realpath } from "node:fs/promises"; -import { basename, extname, isAbsolute, relative, resolve, sep } from "node:path"; +import { isAbsolute, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { parseDocument } from "yaml"; @@ -48,7 +48,7 @@ export function validateRemoteSource(source, name = "remote plugin") { assert.ok(!unknown.length, fail(`unknown source field ${unknown.join(", ")}. Use only source, repo, path, ref, and sha.`)); assert.equal(source.source, "github", fail('only "source": "github" remote references are supported.')); assert.ok(typeof source.repo === "string" && /^[A-Za-z0-9][A-Za-z0-9-]*\/(?!\.{1,2}$)[A-Za-z0-9_.-]+$/.test(source.repo), - fail('"repo" must be OWNER/REPO, such as "github/evals", not a URL.')); + fail('"repo" must be OWNER/REPO, such as "octo-org/plugins", not a URL.')); if (source.path !== undefined) { assert.ok(relativePath(source.path), fail(`"path" must be a directory relative to the repository root, such as "packages/plugin", without a leading "./" or "/", "..", or backslashes. Omit "path" when plugin.json is at the repository root.`)); } @@ -127,12 +127,6 @@ export function evalFilenameMatcher(patterns) { return (name) => regexes.some((regex) => regex.test(name)); } -// Vally reads eval-level params from .params beside the eval. -export function paramsFileName(evalFileName) { - const ext = extname(evalFileName); - return `${basename(evalFileName, ext)}.params${ext}`; -} - const globRegExp = (glob) => new RegExp(`^${glob.replace(/[.+^$|()[\]{}]/g, "\\$&").replaceAll("**/", "\u0000").replaceAll("*", "[^/]*").replaceAll("\u0000", "(?:.*/)?")}$`); // A case's tags as Vally sees them: the eval's top-level tags, with the case's diff --git a/scripts/evals.mjs b/scripts/evals.mjs index 5ab0ad1..8b8ea26 100644 --- a/scripts/evals.mjs +++ b/scripts/evals.mjs @@ -1,11 +1,10 @@ import { isEntrypoint } from "./entrypoint.mjs"; import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; -import { readFile, readdir, writeFile, access } from "node:fs/promises"; +import { readFile, access } from "node:fs/promises"; import { resolve, delimiter, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; -import { stringify } from "yaml"; -import { evalFilenameMatcher, evalFilenames, expectedCases, paramsFileName, parseTagSelectors, readVallyConfig, specsInSuite } from "./catalog.mjs"; +import { expectedCases, parseTagSelectors, readVallyConfig, specsInSuite } from "./catalog.mjs"; import { runCli } from "./problems.mjs"; export const root = fileURLToPath(new URL("../", import.meta.url)); @@ -13,7 +12,6 @@ const cliDirectory = resolve(root, "node_modules/@microsoft/vally-cli"); const manifest = JSON.parse(await readFile(resolve(cliDirectory, "package.json"), "utf8")); export const vallyCliPath = resolve(cliDirectory, typeof manifest.bin === "string" ? manifest.bin : manifest.bin.vally); export const runtimeBridge = resolve(root, "scripts/eval-runtime.js"); -export const generatedHeader = "# Generated by `node scripts/evals.mjs action-params`. Don't edit; it's rewritten on every CI run."; // A string param whose value contains a slash is a repository-relative path. // Other params, such as matrix values, pass through to Vally unchanged. @@ -43,7 +41,7 @@ export function runtimeEnvironment(base = process.env) { } export function vallyArguments(command, args = [], params = evalParams) { - if (!["eval", "lint", "grade"].includes(command)) throw new Error("Use eval, lint, grade, or action-params."); + if (!["eval", "lint", "grade"].includes(command)) throw new Error("Use eval, lint, or grade."); const defaults = command === "eval" ? ["--workers", "1", "--max-retries", "1", "--junit", "--output-dir", "vally-results"] : command === "lint" ? ["plugins", "--eval-spec", "evals", "--strict"] : []; @@ -51,40 +49,12 @@ export function vallyArguments(command, args = [], params = evalParams) { return [vallyCliPath, command, ...defaults, ...flags, ...args]; } -// The Evals Action runs Vally directly and can't pass --param, so CI writes -// eval-level params files (gitignored) holding this runner's absolute paths. -// A params file without the generated header was written by a person, so it's -// never overwritten. -export async function writeActionParams(directory = resolve(root, "evals"), params = evalParams, config) { - const matches = evalFilenameMatcher(evalFilenames(config ?? await readVallyConfig(root))); - const written = []; - for (const entry of await readdir(directory, { withFileTypes: true, recursive: true })) { - if (!entry.isFile() || !matches(entry.name)) continue; - const spec = resolve(entry.parentPath, entry.name); - const text = await readFile(spec, "utf8"); - const used = Object.fromEntries(Object.entries(params).filter(([name]) => text.includes(`\${${name}}`))); - if (!Object.keys(used).length) continue; - const target = resolve(entry.parentPath, paramsFileName(entry.name)); - let existing; - try { existing = await readFile(target, "utf8"); } - catch (error) { if (error.code !== "ENOENT") throw error; } - if (existing !== undefined && !existing.startsWith(generatedHeader)) { - throw new Error(`${relative(root, target)} wasn't generated by action-params, so it won't be overwritten. ` - + `Remove it, or move its params into .vally.yaml, so CI can supply absolute paths for ${Object.keys(used).join(", ")}.`); - } - await writeFile(target, `${generatedHeader}\n${stringify({ params: used })}`); - written.push(target); - } - return written; -} - export const usage = `Usage: node scripts/evals.mjs [options] [Vally options] Commands: eval Run evals with real Copilot sessions. Costs AI credits. lint Check skills and eval specs without a model (npm run eval:lint). grade Re-grade saved evidence (npm run eval:grade). - action-params Write eval params files for the Evals Action in CI. Launcher options for eval and lint: --plugin Select every eval spec for a plugin: the specs under @@ -240,10 +210,6 @@ async function main() { if (!command) process.exitCode = 1; return; } - if (command === "action-params") { - for (const file of await writeActionParams()) console.log(`Wrote ${file}`); - return; - } if (!["eval", "lint", "grade"].includes(command)) throw new Error(`Unknown command ${command}.\n\n${usage}`); const options = launcherOptions(raw); if (options.help) console.log(`${usage}\n\nVally's options for ${command}:\n`); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 13fe62f..ecd010d 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -5,7 +5,7 @@ import { resolve, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { spawnSync } from "node:child_process"; import Ajv from "ajv/dist/2020.js"; -import { contained, frontmatter, parseYaml, readCatalog, readVallyConfig, remoteWarnings } from "./catalog.mjs"; +import { contained, frontmatter, readCatalog, readVallyConfig, remoteWarnings } from "./catalog.mjs"; import { Problems, runCli } from "./problems.mjs"; export const root = fileURLToPath(new URL("../", import.meta.url)); @@ -148,18 +148,6 @@ export async function staleReferenceWarnings(directory, catalog, allFiles) { } } warnings.push(...await adoptionWarnings(directory, catalog, allFiles)); - // Debugging a run with transcripts is fine, but they can hold prompts, model - // output, and code, so a merged workflow shouldn't keep uploading them. - const evalsWorkflow = await read(resolve(directory, ".github/workflows/evals.yml")); - if (evalsWorkflow) { - let parsed; - try { parsed = parseYaml(evalsWorkflow, ".github/workflows/evals.yml"); } catch { parsed = undefined; } - for (const [name, job] of Object.entries(parsed?.jobs ?? {})) { - if ((job?.steps ?? []).some((step) => String(step?.with?.["upload-transcripts"]) === "true")) { - warnings.push(`.github/workflows/evals.yml: job ${name} uploads transcripts. Set upload-transcripts back to false once you've finished debugging; see docs/ci.md.`); - } - } - } return warnings; } diff --git a/test/eval-setup.test.mjs b/test/eval-setup.test.mjs index 66f9b31..2718c03 100644 --- a/test/eval-setup.test.mjs +++ b/test/eval-setup.test.mjs @@ -6,9 +6,9 @@ import { existsSync } from "node:fs"; import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { parse } from "yaml"; import { checkCoverage, discoverAssets, readEvals, root, suiteTags } from "../scripts/validate-evals.mjs"; -import { evalFilenameMatcher, paramsFileName, parseTagSelectors, readCatalog, readVallyConfig } from "../scripts/catalog.mjs"; +import { evalFilenameMatcher, parseTagSelectors, readCatalog, readVallyConfig } from "../scripts/catalog.mjs"; import { defaultRuntime, pluginDirectories, runtimeArguments, runtimeEntry } from "../scripts/eval-runtime.js"; -import { evalParams, generatedHeader, isPathParam, pathParams, reportArguments, runtimeBridge, runtimeEnvironment, vallyArguments, vallyCliPath, vallySelection, writeActionParams } from "../scripts/evals.mjs"; +import { evalParams, isPathParam, pathParams, reportArguments, runtimeBridge, runtimeEnvironment, vallyArguments, vallyCliPath, vallySelection } from "../scripts/evals.mjs"; import { expectedCases, render, summarize } from "../scripts/eval-report.mjs"; test("all local assets have positive, smoke, and negative coverage", async () => { @@ -116,37 +116,7 @@ test("path params come only from .vally.yaml, so removing an example removes its } }); -test("Action params files hold only referenced absolute params and never overwrite authored files", async () => { - const directory = await mkdtemp(join(tmpdir(), "starter-action-params-")); - const params = { JSON_MCP_SERVER: resolve(directory, "server.mjs") }; - const config = { paths: { evalFilenames: ["eval.yaml", "*.eval.yaml"] } }; - try { - await mkdir(join(directory, "uses"), { recursive: true }); - await mkdir(join(directory, "plain"), { recursive: true }); - await mkdir(join(directory, "upstream"), { recursive: true }); - await writeFile(join(directory, "uses/eval.yaml"), "stimuli:\n - args: ['${JSON_MCP_SERVER}']\n"); - await writeFile(join(directory, "plain/eval.yaml"), "stimuli: []\n"); - await writeFile(join(directory, "upstream/release.eval.yaml"), "stimuli:\n - args: ['${JSON_MCP_SERVER}']\n"); - const written = (await writeActionParams(directory, params, config)).sort(); - assert.deepEqual(written, [join(directory, "upstream/release.eval.params.yaml"), join(directory, "uses/eval.params.yaml")].sort()); - for (const file of written) { - const text = await readFile(file, "utf8"); - assert.ok(text.startsWith(generatedHeader)); - assert.deepEqual(parse(text), { params }); - } - assert.equal((await writeActionParams(directory, params, config)).length, 2); - await writeFile(join(directory, "uses/eval.params.yaml"), "params:\n MODEL: [a, b]\n"); - await assert.rejects(writeActionParams(directory, params, config), /wasn't generated by action-params/); - assert.equal(await readFile(join(directory, "uses/eval.params.yaml"), "utf8"), "params:\n MODEL: [a, b]\n"); - } finally { - await rm(directory, { recursive: true, force: true }); - } - const ignore = await readFile(resolve(root, ".gitignore"), "utf8"); - assert.match(ignore, /^evals\/\*\*\/eval\.params\.yaml$/m); - assert.match(ignore, /^evals\/\*\*\/\*\.eval\.params\.yaml$/m); -}); - -test("eval discovery follows .vally.yaml evalFilenames, including the upstream *.eval.yaml convention", async () => { +test("eval discovery follows .vally.yaml evalFilenames, including a custom *.eval.yaml pattern", async () => { const directory = await mkdtemp(join(tmpdir(), "starter-eval-names-")); try { await mkdir(join(directory, "evals/a"), { recursive: true }); @@ -161,7 +131,6 @@ test("eval discovery follows .vally.yaml evalFilenames, including the upstream * assert.deepEqual(await names({}), ["a"]); assert.deepEqual(await readEvals(directory, { paths: { evalFilenames: ["missing.yaml"] } }), []); assert.throws(() => evalFilenameMatcher(["evals/**/eval.yaml"]), /plain file names/); - assert.equal(paramsFileName("release.eval.yaml"), "release.eval.params.yaml"); } finally { await rm(directory, { recursive: true, force: true }); } @@ -221,7 +190,7 @@ test("the source validator accepts CRLF changelog headings and excludes generate // the customizations docs/ci.md describes keep npm run verify passing: trimming // the validate matrix, removing the evals environment, uploading transcripts // for a debug run, or deleting evals.yml. -test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals Action or the npm runner on trusted code", async () => { +test("deterministic CI stays model-free, and opt-in behavioral CI runs the pinned Vally launcher on trusted code", async () => { const automatic = parse(await readFile(resolve(root, ".github/workflows/validate.yml"), "utf8")); const evalsPath = resolve(root, ".github/workflows/evals.yml"); const evals = existsSync(evalsPath) ? parse(await readFile(evalsPath, "utf8")) : undefined; @@ -255,8 +224,7 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals assert.ok(!("pull_request_target" in evals.on)); assert.deepEqual(evals.permissions, { contents: "read" }); - const runners = Object.entries(evals.jobs).filter(([, job]) => job.steps.some((step) => - step.uses?.startsWith("github/evals-action@") || /scripts\/evals\.mjs eval\b/.test(step.run ?? ""))); + const runners = Object.entries(evals.jobs).filter(([, job]) => job.steps.some((step) => /scripts\/evals\.mjs eval\b/.test(step.run ?? ""))); assert.ok(runners.length, "evals.yml should have a job that runs evals."); for (const [name, job] of runners) { // An environment is optional (docs/ci.md explains when to drop it); if @@ -276,36 +244,12 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals const preflight = job.steps.find((step) => step.name === "Check the runtime bridge"); if (preflight) assert.ok(job.steps.indexOf(job.steps.find((step) => step.run?.startsWith("npm ci"))) < job.steps.indexOf(preflight), name); } - if (evals.jobs["check-runner"]) { - // A mistyped runner fails without a token, instead of falling through to a runner. - assert.deepEqual(evals.jobs["check-runner"].permissions, {}); - assert.equal(evals.jobs["check-runner"].if, "vars.EVALS_RUNNER != '' && vars.EVALS_RUNNER != 'action' && vars.EVALS_RUNNER != 'npm'"); - } - // Only one runner job can match a given EVALS_RUNNER. - if (evals.jobs.evals && evals.jobs["evals-npm"]) { - assert.match(evals.jobs.evals.if, /^\(vars\.EVALS_RUNNER == '' \|\| vars\.EVALS_RUNNER == 'action'\) &&/); - assert.match(evals.jobs["evals-npm"].if, /^vars\.EVALS_RUNNER == 'npm' &&/); - } - const job = evals.jobs.evals; - if (job) { - const steps = job.steps; - const action = steps.find((step) => step.uses?.startsWith("github/evals-action@")); - assert.ok(steps.indexOf(steps.find((step) => step.run?.startsWith("npm ci"))) < steps.indexOf(action)); - assert.ok(steps.indexOf(steps.find((step) => step.run === "node scripts/evals.mjs action-params")) < steps.indexOf(action)); - assert.equal(action.with["vally-version"], manifest.devDependencies["@microsoft/vally-cli"]); - assert.equal(action.with["copilot-token"], "${{ github.token }}"); - // Naming anything in credential-env turns off token injection; only these two non-secrets go through it. - assert.deepEqual(action.with["credential-env"].trim().split("\n"), ["COPILOT_CLI_PATH", "VALLY_TELEMETRY_OPTOUT"]); - assert.equal(action.env.COPILOT_CLI_PATH, "${{ github.workspace }}/scripts/eval-runtime.js"); - } - const npmJob = evals.jobs["evals-npm"]; - if (npmJob) { - assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the Action."); - const runStep = npmJob.steps.find((step) => /scripts\/evals\.mjs eval\b/.test(step.run ?? "")); - assert.ok(runStep.run.includes("--require-pass")); - const report = npmJob.steps.find((step) => /eval-report\.mjs/.test(step.run ?? "")); - if (report) assert.ok(npmJob.steps.indexOf(runStep) < npmJob.steps.indexOf(report)); - const upload = npmJob.steps.find((step) => step.uses?.startsWith("actions/upload-artifact@")); + for (const [name, job] of runners) { + const runStep = job.steps.find((step) => /scripts\/evals\.mjs eval\b/.test(step.run ?? "")); + assert.ok(runStep.run.includes("--require-pass"), name); + const report = job.steps.find((step) => /eval-report\.mjs/.test(step.run ?? "")); + if (report) assert.ok(job.steps.indexOf(runStep) < job.steps.indexOf(report), name); + const upload = job.steps.find((step) => step.uses?.startsWith("actions/upload-artifact@")); if (upload) assert.match(upload.if, /vars\.EVALS_UPLOAD_RESULTS == 'true'/, "Transcripts upload only when someone opts in."); } }); diff --git a/test/marketplace.test.mjs b/test/marketplace.test.mjs index f1c12ea..0be91da 100644 --- a/test/marketplace.test.mjs +++ b/test/marketplace.test.mjs @@ -159,9 +159,7 @@ test("verify warns about scaffold text in every scaffolded file and the template await write("plugins/done/README.md", "# Done\n\nReal instructions.\n\n## Host checks\n\n- Try it in each host.\n"), await write("plugins/unchecked/README.md", "# Unchecked\n\nReal instructions.\n"), ]; - await write(".github/workflows/evals.yml", "jobs:\n evals:\n steps:\n - uses: github/evals-action@x\n with:\n upload-transcripts: true\n"); const warnings = (await staleReferenceWarnings(directory, { name: "ours", owner: { name: "Acme" } }, files)).join("\n"); - assert.match(warnings, /evals\.yml: job evals uploads transcripts/); for (const path of ["plugins/mine/README.md", "plugins/mine/src/server.ts", "plugins/mine/com.github.copilot/hooks/mine.mjs", "plugins/mine/com.github.copilot/extensions/mine/extension.mjs", "evals/mine/eval.yaml"]) { assert.match(warnings, new RegExp(`${path.replaceAll(".", "\\.")}: still has scaffolded text`), path); From cc13a0c882d4d152bf0e3aed382037f6773d8185 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Fri, 30 Oct 2026 10:05:42 -0700 Subject: [PATCH 3/3] Rebuild the JSON configuration server with fast-uri 3.1.9 The bundled MCP server includes fast-uri, which had four open Dependabot advisories (two moderate, two low). Updated it in the lockfile, rebuilt the bundle and its notices, and bumped json-configuration to 0.1.7. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- marketplace.json | 2 +- package-lock.json | 6 +- plugins/json-configuration/CHANGELOG.md | 5 + .../dist/THIRD_PARTY_NOTICES.txt | 2 +- plugins/json-configuration/dist/server.mjs | 94 +++++++++++++++---- plugins/json-configuration/plugin.json | 2 +- 6 files changed, 86 insertions(+), 25 deletions(-) diff --git a/marketplace.json b/marketplace.json index 7e85a9b..88a15ca 100644 --- a/marketplace.json +++ b/marketplace.json @@ -35,7 +35,7 @@ { "name": "json-configuration", "source": "./plugins/json-configuration", - "version": "0.1.6", + "version": "0.1.7", "description": "Validate JSON and retrieve configuration values with a local MCP server." } ] diff --git a/package-lock.json b/package-lock.json index 99550a4..7cfd62f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2780,9 +2780,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", - "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", "dev": true, "funding": [ { diff --git a/plugins/json-configuration/CHANGELOG.md b/plugins/json-configuration/CHANGELOG.md index 2443c79..2850636 100644 --- a/plugins/json-configuration/CHANGELOG.md +++ b/plugins/json-configuration/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.1.7 + +- Rebuild the bundled server with fast-uri 3.1.9, which fixes four URI + parsing advisories. + ## 0.1.6 - Remove the plugin's own `LICENSE`. The plugin is still MIT licensed, under diff --git a/plugins/json-configuration/dist/THIRD_PARTY_NOTICES.txt b/plugins/json-configuration/dist/THIRD_PARTY_NOTICES.txt index f89d4c1..bba2352 100644 --- a/plugins/json-configuration/dist/THIRD_PARTY_NOTICES.txt +++ b/plugins/json-configuration/dist/THIRD_PARTY_NOTICES.txt @@ -102,7 +102,7 @@ SOFTWARE. ======================================================================== -fast-uri@3.1.8 (BSD-3-Clause) +fast-uri@3.1.9 (BSD-3-Clause) Copyright (c) 2011-2021, Gary Court until https://github.com/garycourt/uri-js/commit/a1acf730b4bba3f1097c9f52e7d9d3aba8cdcaae Copyright (c) 2021-present The Fastify team All rights reserved. diff --git a/plugins/json-configuration/dist/server.mjs b/plugins/json-configuration/dist/server.mjs index 310fabd..7a08a66 100644 --- a/plugins/json-configuration/dist/server.mjs +++ b/plugins/json-configuration/dist/server.mjs @@ -3337,6 +3337,37 @@ var require_utils = __commonJS({ } return output2.join(""); } + function removeDotSegmentsRelative(path) { + if (path === "" || path[0] === "/") { + return removeDotSegments(path); + } + const segments = path.split("/"); + const last = segments.length - 1; + const output2 = []; + let trailingDirectory = false; + for (let i = 0; i <= last; i++) { + const segment = segments[i]; + if (segment === "." || segment === "..") { + if (segment === "..") { + if (output2.length !== 0 && output2[output2.length - 1] !== "..") { + output2.pop(); + } else { + output2.push(".."); + } + } + trailingDirectory = i === last; + } else { + output2.push(segment); + } + } + if (trailingDirectory && output2[output2.length - 1] !== "..") { + output2.push(""); + } + if (output2.length === 0 || output2[0] === "" || output2[0].includes(":")) { + output2.unshift("."); + } + return output2.join("/"); + } var HOST_DELIMS = { "@": "%40", "/": "%2F", "?": "%3F", "#": "%23", ":": "%3A" }; var HOST_DELIM_RE = /[@/?#:]/g; var HOST_DELIM_NO_COLON_RE = /[@/?#]/g; @@ -3597,6 +3628,7 @@ var require_utils = __commonJS({ encodeFragment, escapePreservingEscapes, removeDotSegments, + removeDotSegmentsRelative, isIPv4, isUUID, normalizeIPv6, @@ -3820,16 +3852,18 @@ var require_schemes = __commonJS({ var require_fast_uri = __commonJS({ "node_modules/fast-uri/index.js"(exports, module) { "use strict"; - var { normalizeIPv6, removeDotSegments, recomposeAuthority, normalizePercentEncoding, normalizePathEncoding, serializePathEncoding, normalizeQueryFragmentEncoding, encodeQuery, encodeFragment, reescapeHostDelimiters, isIPv4, nonSimpleDomain } = require_utils(); + var { normalizeIPv6, removeDotSegments, removeDotSegmentsRelative, recomposeAuthority, normalizePercentEncoding, normalizePathEncoding, serializePathEncoding, normalizeQueryFragmentEncoding, encodeQuery, encodeFragment, reescapeHostDelimiters, isIPv4, nonSimpleDomain } = require_utils(); var { SCHEMES, getSchemeHandler } = require_schemes(); var VALID_SCHEME = /^[A-Za-z][A-Za-z0-9+.-]*$/u; var MALFORMED_SCHEME_ERROR = "URI scheme is malformed."; - function decodeValidScheme(scheme) { - const decodedScheme = unescape(String(scheme)); - if (!VALID_SCHEME.test(decodedScheme)) { + var MALFORMED_HOST_ERROR = "URI host is malformed."; + var FORBIDDEN_DOMAIN_HOST_CHARACTER = /[\0-\x20"#/:<>?@[\\\]^`{|}\x7F]/u; + function validateScheme(scheme) { + const stringScheme = String(scheme); + if (!VALID_SCHEME.test(stringScheme)) { throw new TypeError(MALFORMED_SCHEME_ERROR); } - return decodedScheme; + return stringScheme; } function normalize(uri, options) { if (typeof uri === "string") { @@ -3866,6 +3900,9 @@ var require_fast_uri = __commonJS({ const resolvedSchemeHandler = getSchemeHandler(options && options.scheme || resolved.scheme); const resolvedHost = resolved.host; const resolvedHostIsIP = resolvedHost !== void 0 && resolvedHost !== "" && (isIPv4(resolvedHost) || normalizeIPv6(resolvedHost).isIPV6); + if (hasForbiddenDomainHost(resolved, options || {}, resolvedSchemeHandler, resolvedHostIsIP)) { + throw new Error(MALFORMED_HOST_ERROR); + } canonicalizeHost(resolved, options || {}, resolvedSchemeHandler, resolvedHostIsIP); const encodedASCIIHost = resolvedHost && resolvedHost.indexOf("%") !== -1 && !new RegExp("\\P{ASCII}", "u").test(resolvedHost); if (resolved.error && !encodedASCIIHost) { @@ -3907,14 +3944,15 @@ var require_fast_uri = __commonJS({ if (relative2.path[0] === "/") { target.path = removeDotSegments(relative2.path); } else { - if ((base.userinfo !== void 0 || base.host !== void 0 || base.port !== void 0) && !base.path) { + const baseHasAuthority = base.userinfo !== void 0 || base.host !== void 0 || base.port !== void 0; + if (baseHasAuthority && !base.path) { target.path = "/" + relative2.path; } else if (!base.path) { target.path = relative2.path; } else { target.path = base.path.slice(0, base.path.lastIndexOf("/") + 1) + relative2.path; } - target.path = removeDotSegments(target.path); + target.path = base.scheme === void 0 && !baseHasAuthority && target.path[0] !== "/" ? removeDotSegmentsRelative(target.path) : removeDotSegments(target.path); } target.query = relative2.query; } @@ -3952,12 +3990,13 @@ var require_fast_uri = __commonJS({ const options = Object.assign({}, opts); const uriTokens = []; if (component.scheme) { - component.scheme = decodeValidScheme(component.scheme); + component.scheme = validateScheme(component.scheme); } const schemeHandler = getSchemeHandler(options.scheme || component.scheme); if (schemeHandler && schemeHandler.serialize) schemeHandler.serialize(component, options); const hasAuthority = component.userinfo !== void 0 || component.host !== void 0 || component.port !== void 0; - const pathNoScheme = !options.skipEscape && component.scheme === void 0 && !hasAuthority; + const schemelessReference = (component.scheme === void 0 || options.reference === "suffix") && !hasAuthority; + const pathNoScheme = !options.skipEscape && schemelessReference; if (component.path !== void 0) { if (!options.skipEscape) { component.path = serializePathEncoding(component.path, pathNoScheme); @@ -3966,7 +4005,7 @@ var require_fast_uri = __commonJS({ } } if (options.reference !== "suffix" && component.scheme) { - component.scheme = decodeValidScheme(component.scheme); + component.scheme = validateScheme(component.scheme); uriTokens.push(component.scheme, ":"); } const authority = recomposeAuthority(component); @@ -3982,9 +4021,9 @@ var require_fast_uri = __commonJS({ if (component.path !== void 0) { let s = component.path; if (!options.absolutePath && (!schemeHandler || !schemeHandler.absolutePath)) { - s = removeDotSegments(s); + s = schemelessReference && s[0] !== "/" ? removeDotSegmentsRelative(s) : removeDotSegments(s); } - if (pathNoScheme) { + if (schemelessReference) { s = serializePathEncoding(s, true); } if (authority === void 0 && s[0] === "/" && s[1] === "/") { @@ -4031,16 +4070,34 @@ var require_fast_uri = __commonJS({ return hasMalformedPercentEncoding(matches[3]) || host !== void 0 && !isIPLiteral(host) && hasMalformedPercentEncoding(host) || hasMalformedPercentEncoding(matches[6]) || hasMalformedPercentEncoding(matches[7]) || hasMalformedPercentEncoding(matches[8]); } function canonicalizeHost(parsed, options, schemeHandler, isIP) { + if (hasForbiddenDomainHost(parsed, options, schemeHandler, isIP)) { + parsed.error = parsed.error || MALFORMED_HOST_ERROR; + return true; + } if (!options.unicodeSupport && (!schemeHandler || !schemeHandler.unicodeSupport) && parsed.host && !isIPLiteral(parsed.host) && (options.domainHost || schemeHandler && schemeHandler.domainHost) && isIP === false && nonSimpleDomain(parsed.host)) { + let url2; try { - parsed.host = new URL("http://" + parsed.host).hostname; + url2 = new URL("http://" + parsed.host); } catch (e) { parsed.error = parsed.error || "Host's domain name can not be converted to ASCII: " + e; return true; } + if (!isHostOnlyURL(url2)) { + parsed.error = parsed.error || MALFORMED_HOST_ERROR; + return true; + } + parsed.host = url2.hostname; } return false; } + function hasForbiddenDomainHost(parsed, options, schemeHandler, isIP) { + return Boolean( + parsed.host && isIP === false && !isIPLiteral(parsed.host) && (options.domainHost || schemeHandler && schemeHandler.domainHost) && FORBIDDEN_DOMAIN_HOST_CHARACTER.test(parsed.host) + ); + } + function isHostOnlyURL(url2) { + return url2.username === "" && url2.password === "" && url2.port === "" && url2.pathname === "/" && url2.search === "" && url2.hash === "" && url2.hostname !== ""; + } function parseWithStatus(uri, opts) { const options = Object.assign({}, opts); const parsed = { @@ -4095,9 +4152,8 @@ var require_fast_uri = __commonJS({ parsed.query = matches[7]; parsed.fragment = matches[8]; if (parsed.scheme !== void 0) { - const decodedScheme = unescape(parsed.scheme); - if (VALID_SCHEME.test(decodedScheme)) { - parsed.scheme = decodedScheme.toLowerCase(); + if (VALID_SCHEME.test(parsed.scheme)) { + parsed.scheme = parsed.scheme.toLowerCase(); } else { parsed.error = parsed.error || MALFORMED_SCHEME_ERROR; malformedScheme = true; @@ -4125,7 +4181,7 @@ var require_fast_uri = __commonJS({ malformedIPLiteral = hasIPLiteralBracket && (!bracketedIPLiteral || ipv6result.error === true); parsed.host = isIP ? ipv6result.host : ipv6result.host.toLowerCase(); if (malformedIPLiteral) { - parsed.error = parsed.error || "URI host is malformed."; + parsed.error = parsed.error || MALFORMED_HOST_ERROR; malformedAuthorityOrPort = true; } } else { @@ -4153,7 +4209,7 @@ var require_fast_uri = __commonJS({ if (!isIP) { host = normalizePercentEncoding(host.toLowerCase()); } - parsed.host = reescapeHostDelimiters(host, isIP); + parsed.host = isIP || options.scheme !== "null" ? reescapeHostDelimiters(host, isIP) : host; } if (!schemeHandler || schemeHandler && !schemeHandler.skipNormalize) { if (parsed.path) { @@ -36714,7 +36770,7 @@ var toolDefinitions = [ ]; var TOOL_NAMES = Object.freeze(toolDefinitions.map((tool) => tool.name)); function createServer() { - const server = new McpServer({ name: "json-configuration", version: "0.1.6" }); + const server = new McpServer({ name: "json-configuration", version: "0.1.7" }); for (const tool of toolDefinitions) tool.register(server); return server; } diff --git a/plugins/json-configuration/plugin.json b/plugins/json-configuration/plugin.json index 53db9e7..5dd6cb9 100644 --- a/plugins/json-configuration/plugin.json +++ b/plugins/json-configuration/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "json-configuration", - "version": "0.1.6", + "version": "0.1.7", "description": "Validate JSON and retrieve configuration values with a local MCP server.", "license": "MIT", "keywords": ["json", "configuration", "mcp"]