From e0c6df3e5b6c3eb343122f84d58d9dcfbebf2473 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:50:48 -0800 Subject: [PATCH 1/3] chore(marketplace): update eval remote pin to copilot-evals-v0.24.41 and remove version churn Update marketplace.json's eval remote plugin pin to copilot-evals-v0.24.41 (9edf65253f9ced590401bd99618d52230fee88b2) and make .vally.yaml, README.md, docs/checks-and-evals.md, and backend-defaults.test.mjs read or link without hardcoded version/SHA churn so future pin updates only touch marketplace.json. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/backend-defaults.test.mjs | 1 - .vally.yaml | 31 +++++++++++++++-------- README.md | 6 ++--- docs/checks-and-evals.md | 4 +-- marketplace.json | 6 ++--- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/scripts/backend-defaults.test.mjs b/.github/scripts/backend-defaults.test.mjs index a7d395c..f13a7e9 100644 --- a/.github/scripts/backend-defaults.test.mjs +++ b/.github/scripts/backend-defaults.test.mjs @@ -31,7 +31,6 @@ function assertDocumentedEval(projectRoot, path) { } test("project root fields match the checked Evals readers and inherit native local execution", async () => { - // Root-key contract for marketplace-pinned Evals 0.24.6 (copilot-evals-v0.24.6). const projectKeys = new Set(["environments", "params", "metadata", "paths", "suites"]); const configuration = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8")); for (const key of Object.keys(configuration)) { diff --git a/.vally.yaml b/.vally.yaml index ac5caad..01dd3fd 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -11,6 +11,9 @@ paths: environments: marketplace: # Vally resolves these paths relative to each eval, not this file. + files: + - src: ../../../../marketplace.json + dest: .vally-eval-pin.json skills: - ../../../../plugins/plugin-authoring/skills/add-plugin - ../../../../plugins/plugin-authoring/skills/try-plugin-locally @@ -20,17 +23,25 @@ environments: - ../../../../plugins/pr-description/skills/pr-description # Vally runs setup in each trial workspace; Copilot discovers these skills natively. commands: - - git clone --filter=blob:none --no-checkout --depth 1 --branch - copilot-evals-v0.24.6 --sparse https://github.com/github/evals.git - .vally-eval-source - - git -C .vally-eval-source sparse-checkout set packages/plugin/skills - - git -C .vally-eval-source checkout --detach - 5c3d5213f88149fe1e5476ef029277fe56fd42e4 - >- - node -e "const fs = require('node:fs'); - fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', - { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true - });" + node -e "const fs = require('node:fs'); const { execFileSync } = + require('node:child_process'); const pin = + JSON.parse(fs.readFileSync('.vally-eval-pin.json', + 'utf8')).plugins.find(p => p.name === 'eval')?.source; if (!pin || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(pin.repo) || + !/^[A-Za-z0-9._/-]+$/.test(pin.ref) || pin.ref.startsWith('-') || + !/^[0-9a-f]{40}$/.test(pin.sha)) { throw new Error('Invalid eval remote + pin in marketplace.json'); } try { execFileSync('git', ['clone', + '--filter=blob:none', '--no-checkout', '--depth', '1', '--branch', + pin.ref, '--sparse', 'https://github.com/' + pin.repo + '.git', + '.vally-eval-source'], { stdio: 'inherit' }); execFileSync('git', ['-C', + '.vally-eval-source', 'sparse-checkout', 'set', + 'packages/plugin/skills'], { stdio: 'inherit' }); execFileSync('git', + ['-C', '.vally-eval-source', 'checkout', '--detach', pin.sha], { stdio: + 'inherit' }); fs.cpSync('.vally-eval-source/packages/plugin/skills', + '.github/skills', { recursive: true }); } finally { + fs.rmSync('.vally-eval-source', { recursive: true, force: true }); + fs.rmSync('.vally-eval-pin.json', { force: true }); }" suites: smoke: description: One representative case for each skill and agent diff --git a/README.md b/README.md index 75b040f..7315ae1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Included local maintenance plugins and optional remote plugins: | [plugin-authoring](plugins/plugin-authoring/) | Scaffold, try locally, and prepare contributions to this plugin marketplace. | | [marketplace-maintainer](plugins/marketplace-maintainer/) | Review contributed plugins, maintain remote pins, and prepare marketplace releases. | | [pr-description](plugins/pr-description/) | Draft structured pull request descriptions from supplied change notes or diffs without inventing unshipped work. | -| [GitHub Copilot Evals (Preview)](https://github.com/github/evals/blob/copilot-evals-v0.24.6/README.md) (`eval`, optional remote) | Create, run, review, and compare evals for your Copilot customizations. | +| [GitHub Copilot Evals (Preview)](https://github.com/github/evals/blob/main/README.md) (`eval`, optional remote) | Create, run, review, and compare evals for your Copilot customizations. | ## Background @@ -102,7 +102,7 @@ For optional GitHub Copilot Evals, install it explicitly: copilot plugin install eval@plugin-marketplace-starter ``` -See its [upstream setup guidance](https://github.com/github/evals/blob/copilot-evals-v0.24.6/README.md#install), +See its [upstream setup guidance](https://github.com/github/evals/blob/main/README.md#install), including experimental mode for Copilot CLI commands and Canvas. For remote-entry checks and the local-directory install limitation, see [Remote pin testing](plugins/marketplace-maintainer/skills/update-remote-pins/references/remote-pins.md#testing-a-remote-entry). @@ -251,7 +251,7 @@ This template and its local plugins use the [MIT License](LICENSE). Keep its below it. Remote plugins retain their upstream licenses. GitHub Copilot Evals is source -available under the [GitHub Limited Use Source License](https://github.com/github/evals/blob/5c3d5213f88149fe1e5476ef029277fe56fd42e4/LICENSE). +available under the [GitHub Limited Use Source License](https://github.com/github/evals/blob/main/LICENSE). It isn't MIT-licensed or open source. ## Maintainers diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index a512d63..69c16bd 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -127,8 +127,8 @@ no CLI, eval or project backend is specified. This is an intentional compatible default, not a Docker failure fallback. Explicit eval selections, suites and the standard-model experiment inherit it. -This marketplace pins the GitHub Copilot Evals plugin **0.24.6** (`copilot-evals-v0.24.6`). Its -[full-engine project reader at the pinned commit](https://github.com/github/evals/blob/5c3d5213f88149fe1e5476ef029277fe56fd42e4/packages/plugin/dist/mcp/server.mjs) +This marketplace pins the GitHub Copilot Evals plugin in [marketplace.json](../marketplace.json). Its +[full-engine project reader](https://github.com/github/evals/blob/main/packages/plugin/dist/mcp/server.mjs) accepts `environments`, `params`, `metadata`, `paths` and `suites`, and rejects the advanced project `defaults` key. The regression check covers this root-key shape, not an installed-version or end-to-end execution test. Tools still diff --git a/marketplace.json b/marketplace.json index 15957e1..c5cb7ea 100644 --- a/marketplace.json +++ b/marketplace.json @@ -28,10 +28,10 @@ "source": "github", "repo": "github/evals", "path": "packages/plugin", - "ref": "copilot-evals-v0.24.6", - "sha": "5c3d5213f88149fe1e5476ef029277fe56fd42e4" + "ref": "copilot-evals-v0.24.41", + "sha": "9edf65253f9ced590401bd99618d52230fee88b2" }, - "version": "0.24.6", + "version": "0.24.41", "description": "GitHub Copilot Evals (Preview): set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saves use normal host permissions and runs require cost/effect approval.", "homepage": "https://github.com/github/evals" } From e7d966a1e24a6e7e694016a6a519b2fef98ce066 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:05:08 -0800 Subject: [PATCH 2/3] fix(evals): allowlist upstream evals repo in .vally.yaml and fix reader doc ref Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vally.yaml | 6 +++--- docs/checks-and-evals.md | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.vally.yaml b/.vally.yaml index 01dd3fd..dc977d4 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -28,12 +28,12 @@ environments: require('node:child_process'); const pin = JSON.parse(fs.readFileSync('.vally-eval-pin.json', 'utf8')).plugins.find(p => p.name === 'eval')?.source; if (!pin || - !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(pin.repo) || - !/^[A-Za-z0-9._/-]+$/.test(pin.ref) || pin.ref.startsWith('-') || + pin.repo !== 'github/evals' || pin.path !== 'packages/plugin' || + !/^copilot-evals-v[0-9]+\.[0-9]+\.[0-9]+$/.test(pin.ref) || !/^[0-9a-f]{40}$/.test(pin.sha)) { throw new Error('Invalid eval remote pin in marketplace.json'); } try { execFileSync('git', ['clone', '--filter=blob:none', '--no-checkout', '--depth', '1', '--branch', - pin.ref, '--sparse', 'https://github.com/' + pin.repo + '.git', + pin.ref, '--sparse', 'https://github.com/github/evals.git', '.vally-eval-source'], { stdio: 'inherit' }); execFileSync('git', ['-C', '.vally-eval-source', 'sparse-checkout', 'set', 'packages/plugin/skills'], { stdio: 'inherit' }); execFileSync('git', diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 69c16bd..7a75069 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -127,8 +127,10 @@ no CLI, eval or project backend is specified. This is an intentional compatible default, not a Docker failure fallback. Explicit eval selections, suites and the standard-model experiment inherit it. -This marketplace pins the GitHub Copilot Evals plugin in [marketplace.json](../marketplace.json). Its -[full-engine project reader](https://github.com/github/evals/blob/main/packages/plugin/dist/mcp/server.mjs) +This marketplace pins the GitHub Copilot Evals plugin in [marketplace.json](../marketplace.json) +(published on [github/evals releases](https://github.com/github/evals/releases)). Its packaged +full-engine project reader (`packages/plugin/dist/mcp/server.mjs` at the `source.sha` commit in +`marketplace.json`) accepts `environments`, `params`, `metadata`, `paths` and `suites`, and rejects the advanced project `defaults` key. The regression check covers this root-key shape, not an installed-version or end-to-end execution test. Tools still From d347d7506b9e93d61b401ee2f34bd096b4619fe0 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:31:53 -0800 Subject: [PATCH 3/3] fix(evals): support optional eval removal, sha-direct fetch, and workbench.mjs doc ref Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vally.yaml | 35 ++++++++++++++++++++--------------- docs/checks-and-evals.md | 5 +++-- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.vally.yaml b/.vally.yaml index dc977d4..6ad408a 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -25,23 +25,28 @@ environments: commands: - >- node -e "const fs = require('node:fs'); const { execFileSync } = - require('node:child_process'); const pin = + require('node:child_process'); try { const entry = JSON.parse(fs.readFileSync('.vally-eval-pin.json', - 'utf8')).plugins.find(p => p.name === 'eval')?.source; if (!pin || - pin.repo !== 'github/evals' || pin.path !== 'packages/plugin' || - !/^copilot-evals-v[0-9]+\.[0-9]+\.[0-9]+$/.test(pin.ref) || - !/^[0-9a-f]{40}$/.test(pin.sha)) { throw new Error('Invalid eval remote - pin in marketplace.json'); } try { execFileSync('git', ['clone', - '--filter=blob:none', '--no-checkout', '--depth', '1', '--branch', - pin.ref, '--sparse', 'https://github.com/github/evals.git', + 'utf8')).plugins?.find(p => p.name === 'eval'); if (entry) { const pin = + entry.source; if (!pin || pin.source !== 'github' || pin.repo !== + 'github/evals' || pin.path !== 'packages/plugin' || typeof pin.sha !== + 'string' || !/^[0-9a-f]{40}$/.test(pin.sha) || (pin.ref !== undefined && + (typeof pin.ref !== 'string' || !/^[A-Za-z0-9._/-]+$/.test(pin.ref) || + pin.ref.startsWith('-')))) { throw new Error('Invalid eval remote pin in + marketplace.json'); } execFileSync('git', ['init', '-q', '.vally-eval-source'], { stdio: 'inherit' }); execFileSync('git', ['-C', - '.vally-eval-source', 'sparse-checkout', 'set', - 'packages/plugin/skills'], { stdio: 'inherit' }); execFileSync('git', - ['-C', '.vally-eval-source', 'checkout', '--detach', pin.sha], { stdio: - 'inherit' }); fs.cpSync('.vally-eval-source/packages/plugin/skills', - '.github/skills', { recursive: true }); } finally { - fs.rmSync('.vally-eval-source', { recursive: true, force: true }); - fs.rmSync('.vally-eval-pin.json', { force: true }); }" + '.vally-eval-source', 'remote', 'add', 'origin', + 'https://github.com/github/evals.git'], { stdio: 'inherit' }); + execFileSync('git', ['-C', '.vally-eval-source', 'sparse-checkout', + 'set', 'packages/plugin/skills'], { stdio: 'inherit' }); + execFileSync('git', ['-C', '.vally-eval-source', 'fetch', + '--filter=blob:none', '--depth', '1', '--no-tags', 'origin', pin.sha], { + stdio: 'inherit' }); execFileSync('git', ['-C', '.vally-eval-source', + 'checkout', '-q', '--detach', 'FETCH_HEAD'], { stdio: 'inherit' }); + fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', + { recursive: true }); } } finally { fs.rmSync('.vally-eval-source', { + recursive: true, force: true }); fs.rmSync('.vally-eval-pin.json', { + force: true }); }" suites: smoke: description: One representative case for each skill and agent diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 7a75069..1e4db7a 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -129,8 +129,9 @@ standard-model experiment inherit it. This marketplace pins the GitHub Copilot Evals plugin in [marketplace.json](../marketplace.json) (published on [github/evals releases](https://github.com/github/evals/releases)). Its packaged -full-engine project reader (`packages/plugin/dist/mcp/server.mjs` at the `source.sha` commit in -`marketplace.json`) +full-engine project reader (`packages/plugin/com.github.copilot/extensions/copilot-evals/workbench.mjs`, +loaded by `packages/plugin/dist/mcp/server.mjs` at the `eval` entry's pinned `source.ref` / +`source.sha` in `marketplace.json`) accepts `environments`, `params`, `metadata`, `paths` and `suites`, and rejects the advanced project `defaults` key. The regression check covers this root-key shape, not an installed-version or end-to-end execution test. Tools still