From 46819c3db5dd981fc96b408ab43aeface6380415 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Tue, 27 Oct 2026 13:53:29 -0700 Subject: [PATCH 1/6] Prepare repository for open source release Add the community files OSPO asks for and remove links to private repositories from what the template copies into new repositories. - Add CODE_OF_CONDUCT.md and SUPPORT.md from the OSPO templates. - Add GitHub's coordinated disclosure channel to SECURITY.md in a template-only block, so adopters keep their own placeholder. - Name @github/code-ai-evals in CODEOWNERS, keeping the adopter examples. - Add status, Background, Requirements, Maintainers, Support, and Acknowledgement to the README. GitHub-specific sections are template-only. - Add the license and Code of Conduct lines to CONTRIBUTING.md. - Repoint the remote Evals plugin from github/evals to github/evals-dist (eval, copilot-evals-v0.20.3) and update its docs and tests. - Make the npm runner the default in evals.yml; the Evals Action is opt-in with EVALS_RUNNER=action until it's public. - Link the public GitHub Copilot Evals docs instead of private repos. - Warn in verify, after a rename, when CODEOWNERS or CODE_OF_CONDUCT.md still names the template's maintainers or contact. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/CODEOWNERS | 12 +++-- .github/workflows/evals.yml | 23 +++++---- .vally.yaml | 4 +- AGENTS.md | 2 +- CODE_OF_CONDUCT.md | 75 ++++++++++++++++++++++++++++++ CONTRIBUTING.md | 11 +++++ README.md | 93 ++++++++++++++++++++++++++++--------- SECURITY.md | 35 ++++++++++++++ SUPPORT.md | 31 +++++++++++++ docs/ci.md | 41 +++++++++------- docs/evals.md | 8 ++-- docs/remote-plugins.md | 32 ++++++------- docs/security.md | 2 +- marketplace.json | 14 +++--- scripts/catalog.mjs | 2 +- scripts/validate.mjs | 10 ++++ test/adoption.test.mjs | 2 +- test/catalog.test.mjs | 6 +-- test/eval-setup.test.mjs | 6 +-- test/marketplace.test.mjs | 21 +++++++++ 20 files changed, 338 insertions(+), 92 deletions(-) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 SUPPORT.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c730935..892a66d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,11 @@ -# Code owners review changes to the paths below. Replace these examples with -# real GitHub users or teams before you ship, then require code owner review in -# a branch protection rule or ruleset. This file alone doesn't enforce review. +# Code owners review changes to the paths below. The template's own owners are +# its maintainers, @github/code-ai-evals. Replace them with real GitHub users or +# teams before you ship, then require code owner review in a branch protection +# rule or ruleset. This file alone doesn't enforce review. # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # +# Examples for a marketplace of your own: +# # * @your-org/ai-customizations # /.github/workflows/ @your-org/ai-customizations-admins # /.github/copilot/settings.json @your-org/ai-customizations-admins @@ -10,3 +13,6 @@ # /evals/ @your-org/ai-customizations # /marketplace.json @your-org/ai-customizations-admins # /plugins// @your-org/ai-customizations-admins + +# This repository is maintained by: +* @github/code-ai-evals diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index e306f9a..9972149 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -2,11 +2,11 @@ name: Behavioral evals # Off by default, because evals run real Copilot sessions and spend AI credits. # Pull requests run evals only after you set the repository variable # ENABLE_EVALS to true. Manual runs work either way. The repository variable -# EVALS_RUNNER picks the job: unset or "action" uses the private -# github/evals-action; "npm" runs the pinned Vally CLI from package.json, for -# organizations without access to that Action. Any other value fails. Both eval -# jobs use the "evals" environment, so you can require a reviewer to approve -# each run before it gets a token. See docs/ci.md. +# EVALS_RUNNER picks the job: unset or "npm" runs the pinned Vally CLI from +# package.json, which works in any repository; "action" uses the GitHub Copilot +# Evals Action (github/evals-action), which isn't public yet. Any other value +# fails. Both eval jobs use the "evals" environment, so you can require a +# reviewer to approve each run before it gets a token. See docs/ci.md. on: pull_request: types: [opened, synchronize, reopened, ready_for_review] @@ -41,13 +41,13 @@ jobs: env: EVALS_RUNNER: ${{ vars.EVALS_RUNNER }} - # Runner "action". A separate job, because a job downloads every action it - # names before it starts, even ones in skipped steps. + # Runner "action", opt-in. A separate job, because a job downloads every + # action it names before it starts, even ones in skipped steps. evals: # Manual runs always run. Pull requests run only when ENABLE_EVALS is true, # the branch is in this repository (never a fork), and the PR isn't a draft. if: >- - (vars.EVALS_RUNNER == '' || vars.EVALS_RUNNER == 'action') && (github.event_name == 'workflow_dispatch' || + vars.EVALS_RUNNER == 'action' && (github.event_name == 'workflow_dispatch' || (vars.ENABLE_EVALS == 'true' && github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft)) @@ -85,6 +85,8 @@ jobs: run: node "$COPILOT_CLI_PATH" --version env: COPILOT_CLI_PATH: ${{ github.workspace }}/scripts/eval-runtime.js + # TODO: github/evals-action isn't public yet. When it is, move this pin to + # the release that ships its third-party notices. - uses: github/evals-action@83fa12332c3e44ad899f347eaf153a0d9b6432dc # v0.1.0 env: # Vally's SDK executor launches this bridge, which runs the pinned full runtime. @@ -107,10 +109,11 @@ jobs: max-retries: 1 upload-transcripts: false - # Runner "npm": the same launcher contributors use locally, with the job token. + # Runner "npm", the default: the same launcher contributors use locally, with + # the job token. evals-npm: if: >- - vars.EVALS_RUNNER == 'npm' && (github.event_name == 'workflow_dispatch' || + (vars.EVALS_RUNNER == '' || vars.EVALS_RUNNER == 'npm') && (github.event_name == 'workflow_dispatch' || (vars.ENABLE_EVALS == 'true' && github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft)) diff --git a/.vally.yaml b/.vally.yaml index 33de6d7..998df46 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -10,8 +10,8 @@ paths: - plugins/json-configuration/skills evals: - evals - # eval.yaml is this repository's convention; *.eval.yaml is the upstream - # github/evals convention. Both work. + # eval.yaml is this repository's convention; *.eval.yaml is the GitHub + # Copilot Evals convention (https://aka.ms/evals/site). Both work. evalFilenames: - eval.yaml - "*.eval.yaml" diff --git a/AGENTS.md b/AGENTS.md index 9465fdb..cf7de81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,7 +105,7 @@ Consult the relevant contract before changing package or catalog fields: - [Creating a team marketplace](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/plugins-marketplace). - [CLI plugin and catalog reference](https://docs.github.com/en/copilot/reference/copilot-cli-reference/cli-plugin-reference). - [Vally eval schema](https://microsoft.github.io/vally/reference/eval-spec/) and - [Evals Action](https://github.com/github/evals-action). + [GitHub Copilot Evals docs](https://aka.ms/evals/site). The README's [references](README.md#references) cover skills, hooks, VS Code, and this repository's guides. \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..baf4348 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,75 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db76b6a..13324cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,17 @@ Thanks for improving this marketplace. This guide is the checklist for people; [AGENTS.md](AGENTS.md) adds rules for coding agents, and the [README](README.md) explains how everything fits together. +Contributions to this project are +[released](https://docs.github.com/en/site-policy/github-terms/github-terms-of-service#6-contributions-under-repository-license) +to the public under the [project's open source license](LICENSE). This project +is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By +participating in it, you agree to abide by its terms. + + +You may need to agree to a contributor agreement before your pull request can +be merged. + + ## Before you start - Install Node.js 22.14 or newer and run `npm ci`. diff --git a/README.md b/README.md index ddaf078..6895768 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,11 @@ A GitHub template for building, evaluating, and sharing AI customizations works for a team in an organization and for one person with a personal account. + +**Status:** active development. GitHub maintains this template; see +[Maintainers](#maintainers) and [Support](#support). + + It gives you: @@ -38,15 +43,41 @@ Two terms come up a lot: | [code-review](plugins/code-review/) | "Use adversarial-review to review this diff without changing it" | A skill that delegates to a read-only custom agent | | [json-configuration](plugins/json-configuration/) | 'Use json-configuration to validate {"port":8443} and look up /port' | A skill backed by a bundled local MCP server | - + The catalog also has an optional reference to the GitHub Copilot Evals plugin, which lives in another repository. See [Remote plugins](docs/remote-plugins.md). - + [Examples](docs/examples.md) explains what each example shows and which files it owns. + +## Background + +Most shared Copilot customizations ship as loose files, without a catalog, +checks, or evals, because teams have to piece that setup together from docs. +This template puts the whole setup in one repository, so a new skill, agent, +hook, or MCP server arrives with a catalog entry, a model-free check, and an +eval that shows it works. + +Planned work and known gaps are in the +[open issues](https://github.com/github/plugin-marketplace-starter/issues). +Contributions are welcome, from fixes to the scripts and docs to better +examples and new checks. Read [CONTRIBUTING.md](CONTRIBUTING.md) first. + + +## Requirements + +- To install and use the plugins: a current Copilot CLI, the GitHub Copilot + app, or VS Code with agent plugins turned on, and a GitHub Copilot plan. See + [Install plugins](#install-plugins). +- To build, check, and change the marketplace: Node.js 22.14 or newer and npm. + `npm ci` installs everything else, including the pinned Vally CLI and Copilot + runtime that evals use. +- To run behavioral evals: a Copilot plan or organization policy that allows + the usage, since every run spends AI credits. See [Evals](#evals). + ## Make it yours You need Node.js 22.14 or newer. @@ -73,20 +104,22 @@ You need Node.js 22.14 or newer. 4. Add your first plugin with `npm run new-plugin -- `. See [Add a plugin](#add-a-plugin). `npm run verify` passes even with no plugins at all. - -5. Decide whether to keep the remote `github-copilot` plugin in the catalog. It - needs access to the private `github/evals` repository; see - [Remote plugins](docs/remote-plugins.md). Remove it with - `npm run remove-plugin -- github-copilot`. - + +5. Decide whether to keep the remote `eval` plugin, GitHub Copilot Evals, in + the catalog. It installs from `github/evals-dist`, which isn't public yet; + see [Remote plugins](docs/remote-plugins.md). Remove it with + `npm run remove-plugin -- eval`. + 6. Replace what's meant for you before you ship: - Your own copyright line in [LICENSE](LICENSE), beside the template's. That one file covers the whole repository, including every plugin. See [Licensing](#licensing). - The title and introduction of this README. - - Real owners in [.github/CODEOWNERS](.github/CODEOWNERS). On a personal - account, that can be just you. - - Your reporting channel in [SECURITY.md](SECURITY.md). + - Real owners in [.github/CODEOWNERS](.github/CODEOWNERS), in place of the + template's maintainers. On a personal account, that can be just you. + - Your reporting channel in [SECURITY.md](SECURITY.md), your enforcement + contact in [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md), and how people get + help in [SUPPORT.md](SUPPORT.md). - Your process in [CONTRIBUTING.md](CONTRIBUTING.md), the [pull request template](.github/PULL_REQUEST_TEMPLATE.md), and the [new plugin issue form](.github/ISSUE_TEMPLATE/new-plugin.yml). @@ -130,11 +163,6 @@ copilot plugin install json-configuration@plugin-marketplace-starter Choose only the plugins you need; each one installs on its own. Start a new session after installing. - -This repository is currently private, so installers need a GitHub account with -access to it. - - ### VS Code Run **Chat: Install Plugin from Source** and enter the repository, here @@ -234,11 +262,11 @@ marketplace consistent. [evals.yml](.github/workflows/evals.yml) runs behavioral evals. It's off for pull requests until you set the repository variable `ENABLE_EVALS` to `true`, and you can always run it by hand. Each run can wait for a reviewer's approval -through the `evals` environment. By default it uses the private -[`github/evals-action`](https://github.com/github/evals-action), which only -repositories in the GitHub organization can use. Anywhere else, including a -personal account, set the repository variable `EVALS_RUNNER` to `npm` to run -the same evals with the pinned Vally CLI. On a personal account, usage is +through the `evals` environment. By default it runs the same evals with the +pinned Vally CLI and the job's token, so it works in any repository, including +one on a personal account. Repositories that can use the GitHub Copilot Evals +Action, which isn't public yet, can set the repository variable `EVALS_RUNNER` +to `action` instead. On a personal account, usage is billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers setup, personal accounts, billing, cost, and security. @@ -291,6 +319,27 @@ The `Copyright` line in `LICENSE` says who holds the copyright in the code. [license](schemas/LICENSE-APACHE-2.0.txt) and [notice](schemas/NOTICE), and the third-party notices beside each generated bundle. + +## Maintainers + +GitHub's Copilot Evals team, `@github/code-ai-evals`, maintains this template. +[.github/CODEOWNERS](.github/CODEOWNERS) lists the code owners who review +changes. + +## Support + +See [SUPPORT.md](SUPPORT.md) for how to ask questions and report bugs, and +[SECURITY.md](SECURITY.md) for how to report a vulnerability privately. + +## Acknowledgement + +This template builds on the [Agent Plugins](https://agent-plugins.org/) +standard and its schemas, and runs its evals with +[Vally](https://microsoft.github.io/vally/). Thanks to +[@digitarald](https://github.com/digitarald), who wrote its first version, and +to everyone who's contributed since. + + ## References - [Customization types](docs/customization-types.md), [Hooks](docs/hooks.md), @@ -310,4 +359,4 @@ The `Copyright` line in `LICENSE` says who holds the copyright in the code. - [VS Code agent plugins](https://code.visualstudio.com/docs/agent-customization/agent-plugins) - [Agent Skills specification](https://agentskills.io/specification) - [Vally](https://microsoft.github.io/vally/) and the - [Evals Action](https://github.com/github/evals-action) + [GitHub Copilot Evals docs](https://aka.ms/evals/site) diff --git a/SECURITY.md b/SECURITY.md index c8da47f..e53a14b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,6 +4,41 @@ Don't report security problems in public issues, discussions, or pull requests. + +### In this template + +Thanks for helping make GitHub safe for everyone. GitHub takes the security of +its software products and services seriously, including all of the open source +code repositories managed through its GitHub organizations, such as +[GitHub](https://github.com/GitHub). + +If you believe you've found a security vulnerability in this template, +`github/plugin-marketplace-starter`, or in any other GitHub-owned repository, +report it to GitHub through coordinated disclosure. Send an email to +opensource-security[@]github.com. + +Include as much of the following as you can. It helps GitHub understand and +triage your report more quickly. + +- The type of issue, such as command injection, path traversal, or cross-site + scripting +- Full paths of the source files related to the issue +- The location of the affected source code: a tag, branch, commit, or direct + URL +- Any special configuration needed to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Proof-of-concept or exploit code, if possible +- The impact of the issue, including how an attacker might exploit it + +Open source repositories are +[outside the scope of GitHub's bug bounty program](https://bounty.github.com/index.html#scope), +so they aren't eligible for bounty rewards, but GitHub will make sure your +finding reaches the right maintainers. See +[GitHub's Safe Harbor Policy](https://docs.github.com/en/site-policy/security-policies/github-bug-bounty-program-legal-safe-harbor#1-safe-harbor-terms). + +### In a marketplace made from this template + + Use this repository's **Security** tab to [report a vulnerability privately](https://docs.github.com/en/code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository), diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..314faf1 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,31 @@ +# Support + +## How to file issues and get help + +This project uses GitHub issues to track bugs and feature requests. Search the +existing issues before you file a new one, to avoid duplicates. For a new bug +or feature request, file a new issue. To propose a plugin, use the +**New plugin** issue form. + +For help or questions about using this marketplace, open an issue or ask the +maintainers listed in [.github/CODEOWNERS](.github/CODEOWNERS). + +Don't report security problems in public issues. Follow +[SECURITY.md](SECURITY.md) instead. + + +For questions about GitHub Copilot Evals itself, see the +[GitHub Copilot Evals docs](https://aka.ms/evals/site). For problems with +Copilot CLI, the GitHub Copilot app, or the GitHub Copilot Chat VS Code +extension, rather than with this template, use +[GitHub Community discussions](https://github.com/orgs/community/discussions/categories/copilot-conversations) +or [GitHub Support](https://support.github.com/). + +This template is under active development and maintained by GitHub staff and +the community. We'll do our best to respond to support, feature requests, and +community questions in a timely manner. + + +## Support policy + +Support for this project is limited to the resources listed above. diff --git a/docs/ci.md b/docs/ci.md index 48588ad..5a2e5ab 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -39,9 +39,9 @@ organization: - `validate.yml` works as it is, and it's all you need to keep the marketplace consistent. It never calls a model. -- To run behavioral evals in CI, set the repository variable `EVALS_RUNNER` to - `npm`. The default runner uses a private Action that only repositories in the - GitHub organization can use, so without this the eval job fails. +- To run behavioral evals in CI, leave the repository variable `EVALS_RUNNER` + unset or set it to `npm`. The `action` runner uses an Action that isn't + public yet, so it fails outside the GitHub organization. - Copilot usage from the job's token is [billed to your own Copilot plan](https://docs.github.com/en/copilot/concepts/agent-capabilities/agent-powered-workflows-and-operations/copilot-cli-in-github-actions). Each plan includes a monthly allowance of AI credits; see @@ -69,9 +69,10 @@ turn it on. ### Turn it on 1. Check the [requirements](#requirements) below. -2. Pick a runner (see the next section). If you use the npm runner, create the - repository variable `EVALS_RUNNER` with the value `npm`. Any value other - than `action`, `npm`, or nothing fails the workflow instead of guessing. +2. Pick a runner (see the next section). The npm runner is the default. To use + the Evals Action instead, create the repository variable `EVALS_RUNNER` + with the value `action`. Any value other than `action`, `npm`, or nothing + fails the workflow instead of guessing. 3. Decide who approves runs. Both eval jobs use an environment named `evals`, which GitHub creates the first time a job uses it. To make a person approve each run before it gets a token, add required reviewers to it in @@ -96,23 +97,27 @@ Pull request runs start only when all of these hold: - It touches plugins, evals, schemas, scripts, `.vally.yaml`, the package files, or the workflow itself. -Runs don't cancel each other, because the Evals Action requires that, so each -push to an open pull request queues one more run. Keep work-in-progress pull -requests as drafts. +Runs don't cancel each other, which the Evals Action requires, so each push to +an open pull request queues one more run. Keep work-in-progress pull requests +as drafts. ### Pick a runner -| | `action` (default) | `npm` | +| | `npm` (default) | `action` | |---|---|---| -| Set `EVALS_RUNNER` to | nothing, or `action` | `npm` | -| Runs | [`github/evals-action`](https://github.com/github/evals-action) v0.1.0 | `node scripts/evals.mjs eval --suite --require-pass` | -| Who can use it | Repositories in the GitHub organization. The Action is private and shared only there, so other organizations can't use it. | Any repository | -| Report | Structured report and one sticky pull request comment | The job summary | -| Job token permissions | `contents: read`, `copilot-requests: write`, `pull-requests: write` | `contents: read`, `copilot-requests: write` | +| Set `EVALS_RUNNER` to | nothing, or `npm` | `action` | +| Runs | `node scripts/evals.mjs eval --suite --require-pass` | The GitHub Copilot Evals Action, `github/evals-action` v0.1.0 | +| Who can use it | Any repository | Repositories in the GitHub organization, for now. The Action isn't public yet, so other organizations can't use it. | +| Report | The job summary | Structured report and one sticky pull request comment | +| Job token permissions | `contents: read`, `copilot-requests: write` | `contents: read`, `copilot-requests: write`, `pull-requests: write` | + + The two runners are separate jobs, because a job downloads every action it -names before it starts, even ones in skipped steps. With `EVALS_RUNNER=npm`, the -private Action is never downloaded. +names before it starts, even ones in skipped steps. With the npm runner, the +Evals Action is never downloaded. Both fail the job on a failed verdict, an incomplete run, or an infrastructure error, unlike the report-only default of `npm run eval:report` locally. Both use @@ -175,7 +180,7 @@ repository variable `EVALS_UPLOAD_RESULTS` to `true`. Both jobs use `ubuntu-latest`. - **Actions policy.** If your organization restricts which actions can run, allow the actions the workflow pins: `actions/checkout`, `actions/setup-node`, - `actions/upload-artifact`, and, for the default runner, `github/evals-action`. + `actions/upload-artifact`, and, for the `action` runner, `github/evals-action`. ## Cost diff --git a/docs/evals.md b/docs/evals.md index 2fe5800..3111b73 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -84,8 +84,8 @@ and check your billing dashboard after a run to learn what your suite costs. ## Write an eval 1. Create `evals//eval.yaml` with realistic tasks, expected outcomes, - and at least one negative or non-trigger case. The upstream - [github/evals](https://github.com/github/evals) convention, + and at least one negative or non-trigger case. The + [GitHub Copilot Evals](https://aka.ms/evals/site) convention, `.eval.yaml`, works too; `evalFilenames` in `.vally.yaml` lists both. Keep private production data out of fixtures and expected answers out of agent-visible inputs. Put grader programs and answer keys in @@ -218,8 +218,8 @@ them. Test them by hand in the hosts you support. ## Suites Suites are conventions, not fixed names. This repository's `smoke` suite -selects cases tagged `suite: smoke`, and `full` runs every spec. Upstream -github/evals examples filter on `lane: ci` instead. A `tags.suite` value must +selects cases tagged `suite: smoke`, and `full` runs every spec. GitHub Copilot +Evals examples filter on `lane: ci` instead. A `tags.suite` value must match a suite filter in `.vally.yaml`, and every asset needs a case in `smoke` because CI runs it by default. diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index f89180d..4cf0242 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -11,23 +11,24 @@ run upstream code, and the local evals don't cover remote plugins: `evals/` is for the plugins under `plugins/`, and the coverage check rejects asset IDs that aren't local. Evaluate an upstream plugin in its own repository. - + ## The GitHub Copilot Evals plugin -[marketplace.json](../marketplace.json) includes a `github-copilot` entry that -points at the published GitHub Copilot Evals plugin in -[github/evals](https://github.com/github/evals). It mirrors the stable entry in -upstream's own catalog: the `copilot-evals-v0.16.0` release, pinned to the full -SHA of its built commit. + +[marketplace.json](../marketplace.json) includes an `eval` entry that points at +the published GitHub Copilot Evals plugin in `github/evals-dist`. It mirrors +the entry in upstream's own catalog: the `copilot-evals-v0.20.3` release, +pinned to the full SHA of its commit. After adding this marketplace, install it explicitly with -`copilot plugin install github-copilot@MARKETPLACE`, where `MARKETPLACE` is the +`copilot plugin install eval@MARKETPLACE`, where `MARKETPLACE` is the `name` in `marketplace.json`. Or skip this catalog and install it from -upstream, which follows upstream's reviewed stable releases: +upstream, which follows upstream's releases: ```sh -copilot plugin marketplace add github/evals -copilot plugin install github-copilot@evals +copilot plugin marketplace add github/evals-dist +copilot plugin install eval@github-copilot-evals-dist ``` Pick one route. Don't install the same plugin from both marketplaces. @@ -36,15 +37,14 @@ The pinned release is the built plugin. It provides the native `/eval` command (`/eval setup`, `create`, `review`, `improve`, `run`, and `compare`) in the CLI and Copilot App, a canvas, and six model-invocable skills. Running an eval needs a compatible runner and its own approval, and it spends AI credits. See -the [upstream install guide](https://github.com/github/evals/blob/main/website/src/content/guides/install/plugin.md) -for VS Code and Copilot App setup. This repository's eval tooling works without -it. +the [GitHub Copilot Evals docs](https://aka.ms/evals/site) for VS Code and +Copilot App setup. This repository's eval tooling works without it. -The `github/evals` repository is currently private, so each installer needs a +The `github/evals-dist` repository isn't public yet, so each installer needs a GitHub account with access. This catalog entry doesn't grant access or copy upstream code. If your team doesn't have access, remove the entry with -`npm run remove-plugin -- github-copilot`. - +`npm run remove-plugin -- eval`. + ## Add a remote plugin diff --git a/docs/security.md b/docs/security.md index 8484d20..4d56716 100644 --- a/docs/security.md +++ b/docs/security.md @@ -32,7 +32,7 @@ plugins. | Local eval results in `vally-results/`: results, a report, JUnit, and session evidence with transcripts | Your disk | Gitignored. Delete them when you're done. | | npm runner in CI: Vally's log and the report in the job summary | The workflow run | Anyone who can read the repository's Actions runs. Logs follow the repository's [retention setting](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization) (90 days by default). | | npm runner results artifact, which includes transcripts | A workflow artifact | Only when `EVALS_UPLOAD_RESULTS` is `true`. Kept for 7 days. | -| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. The Action is private, and this template doesn't document its internals. | +| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. The Action isn't public yet, and this template doesn't document its internals. | | Vally telemetry | Vally's telemetry service, unless turned off | `VALLY_TELEMETRY_OPTOUT=1` is set by `validate.yml`, the local launcher, and both eval jobs. The Action job also forwards it to Vally's run phase; whether the Action's lint and report phases honor it isn't verified. | | Copilot CLI telemetry | GitHub | Set by Copilot's own settings and policies, not by this template. | diff --git a/marketplace.json b/marketplace.json index 754e01e..605d20f 100644 --- a/marketplace.json +++ b/marketplace.json @@ -39,17 +39,17 @@ "description": "Validate JSON and retrieve configuration values with a local MCP server." }, { - "name": "github-copilot", + "name": "eval", "source": { "source": "github", - "repo": "github/evals", - "ref": "copilot-evals-v0.16.0", - "sha": "dd4c5015b5e8fdfcbd17b261d996c85aa92fde5b", + "repo": "github/evals-dist", + "ref": "copilot-evals-v0.20.3", + "sha": "acd4b5029b0791fbe1c4b58e28fa94c580841d69", "path": "packages/plugin" }, - "version": "0.16.0", - "description": "Configure native projects with /eval setup, create and save evals with /eval create, save distinct candidates with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running follow separate host approval policies.", - "homepage": "https://github.com/github/evals#installed-skills" + "version": "0.20.3", + "description": "GitHub Copilot Evals: set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running need separate approval.", + "homepage": "https://aka.ms/evals/site" } ] } diff --git a/scripts/catalog.mjs b/scripts/catalog.mjs index fe3a4e3..05e4335 100644 --- a/scripts/catalog.mjs +++ b/scripts/catalog.mjs @@ -48,7 +48,7 @@ export function validateRemoteSource(source, name = "remote plugin") { assert.ok(!unknown.length, fail(`unknown source field ${unknown.join(", ")}. Use only source, repo, path, ref, and sha.`)); assert.equal(source.source, "github", fail('only "source": "github" remote references are supported.')); assert.ok(typeof source.repo === "string" && /^[A-Za-z0-9][A-Za-z0-9-]*\/(?!\.{1,2}$)[A-Za-z0-9_.-]+$/.test(source.repo), - fail('"repo" must be OWNER/REPO, such as "github/evals", not a URL.')); + fail('"repo" must be OWNER/REPO, such as "my-org/plugins", not a URL.')); if (source.path !== undefined) { assert.ok(relativePath(source.path), fail(`"path" must be a directory relative to the repository root, such as "packages/plugin", without a leading "./" or "/", "..", or backslashes. Omit "path" when plugin.json is at the repository root.`)); } diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 13fe62f..aa6a912 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -180,6 +180,10 @@ export const scaffoldText = Object.freeze([ // repository, plugins included. Its MIT notice stays in anything you keep; see // Licensing in README.md. export const templateLicenseHolder = "GitHub, Inc."; +// The template's own maintainers in .github/CODEOWNERS and its conduct contact +// in CODE_OF_CONDUCT.md. A marketplace made from the template names its own. +export const templateCodeOwners = "@github/code-ai-evals"; +export const templateConductContact = "opensource@github.com"; const read = async (path) => { try { return await readFile(path, "utf8"); } catch (error) { if (error.code === "ENOENT") return undefined; throw error; } }; // Every holder named on a "Copyright" line in a license file. @@ -235,6 +239,12 @@ async function adoptionWarnings(directory, catalog, allFiles) { if (owners !== undefined && !owners.split(/\r?\n/).some((line) => line.trim() && !line.trim().startsWith("#"))) { warnings.push(".github/CODEOWNERS: every rule is commented out. Name real owners, then require their review in a ruleset."); } + if (owners?.split(/\r?\n/).some((line) => !line.trim().startsWith("#") && line.split(/\s+/).includes(templateCodeOwners))) { + warnings.push(`.github/CODEOWNERS: still names the template's maintainers, ${templateCodeOwners}. Name your own owners, then require their review in a ruleset.`); + } + if ((await read(resolve(directory, "CODE_OF_CONDUCT.md")))?.includes(templateConductContact)) { + warnings.push(`CODE_OF_CONDUCT.md: still names the template's contact, ${templateConductContact}. Replace it with your own enforcement contact.`); + } return warnings; } diff --git a/test/adoption.test.mjs b/test/adoption.test.mjs index 9b67242..b72f627 100644 --- a/test/adoption.test.mjs +++ b/test/adoption.test.mjs @@ -187,7 +187,7 @@ test("an empty or remote-only marketplace passes verify, and new plugins scaffol const setup = (await readFile(join(directory, "README.md"), "utf8")).match(/## Make it yours\n[\s\S]*?\n## /)?.[0]; if (setup) { if (present.length) assert.doesNotMatch(setup, /remove-example/); - assert.doesNotMatch(setup, /github-copilot/); + assert.doesNotMatch(setup, /remove-plugin -- eval\b|evals-dist/); const numbers = [...setup.matchAll(/^(\d+)\. /gm)].map((match) => Number(match[1])); assert.deepEqual(numbers, numbers.map((_, index) => index + 1)); } diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs index 947decb..7d07538 100644 --- a/test/catalog.test.mjs +++ b/test/catalog.test.mjs @@ -9,10 +9,10 @@ import { discoverAssets, checkCoverage, root } from "../scripts/validate-evals.m test("the catalog separates local plugins from pinned, opt-in remote references", async () => { const { catalog, localPlugins, remotePlugins } = await readCatalog(root); assert.equal(localPlugins.length + remotePlugins.length, catalog.plugins.length); - const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals"); + const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals-dist"); if (evals) { // The catalog name must match upstream's plugin.json name, not its marketplace name. - assert.equal(evals.name, "github-copilot"); + assert.equal(evals.name, "eval"); assert.equal(evals.source.path, "packages/plugin"); assert.deepEqual(remoteWarnings(evals), []); assert.equal(evals.version, evals.source.ref.replace(/^copilot-evals-v/, "")); @@ -37,7 +37,7 @@ test("an empty catalog is valid, and catalog errors are plain sentences", () => test("remote references must pin a full sha, and errors name the entry", () => { const sha = "a".repeat(40); for (const source of [ - { source: "github", repo: "github/evals", path: "packages/plugin", sha }, + { source: "github", repo: "example/evals", path: "packages/plugin", sha }, { source: "github", repo: "example/plugin", ref: "v1.0.0", sha }, { source: "github", repo: "example/plugin", sha }, ]) assert.doesNotThrow(() => validateRemoteSource(source, "fixture")); diff --git a/test/eval-setup.test.mjs b/test/eval-setup.test.mjs index b437c4c..280b7df 100644 --- a/test/eval-setup.test.mjs +++ b/test/eval-setup.test.mjs @@ -283,8 +283,8 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals } // Only one runner job can match a given EVALS_RUNNER. if (evals.jobs.evals && evals.jobs["evals-npm"]) { - assert.match(evals.jobs.evals.if, /^\(vars\.EVALS_RUNNER == '' \|\| vars\.EVALS_RUNNER == 'action'\) &&/); - assert.match(evals.jobs["evals-npm"].if, /^vars\.EVALS_RUNNER == 'npm' &&/); + assert.match(evals.jobs.evals.if, /^vars\.EVALS_RUNNER == 'action' &&/); + assert.match(evals.jobs["evals-npm"].if, /^\(vars\.EVALS_RUNNER == '' \|\| vars\.EVALS_RUNNER == 'npm'\) &&/); } const job = evals.jobs.evals; if (job) { @@ -300,7 +300,7 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals } const npmJob = evals.jobs["evals-npm"]; if (npmJob) { - assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the private Action."); + assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the Evals Action."); const runStep = npmJob.steps.find((step) => /scripts\/evals\.mjs eval\b/.test(step.run ?? "")); assert.ok(runStep.run.includes("--require-pass")); const report = npmJob.steps.find((step) => /eval-report\.mjs/.test(step.run ?? "")); diff --git a/test/marketplace.test.mjs b/test/marketplace.test.mjs index f1c12ea..384dc67 100644 --- a/test/marketplace.test.mjs +++ b/test/marketplace.test.mjs @@ -144,6 +144,27 @@ test("the license warning keeps the template's MIT notice and asks only for your } }); +test("after a rename, verify warns about the template's code owners and conduct contact", async () => { + const directory = await mkdtemp(join(tmpdir(), "starter-owners-")); + const write = async (path, text) => { await mkdir(join(directory, path, ".."), { recursive: true }); await writeFile(join(directory, path), text); return join(directory, path); }; + const warn = async (catalog = { name: "ours", owner: { name: "Acme" } }) => (await staleReferenceWarnings(directory, catalog, [])).join("\n"); + try { + await write("README.md", "# Ours\n"); + await write(".github/CODEOWNERS", "# Replace @github/code-ai-evals with your own owners.\n* @github/code-ai-evals\n"); + await write("CODE_OF_CONDUCT.md", "Report it to .\n"); + assert.match(await warn(), /\.github\/CODEOWNERS: still names the template's maintainers, @github\/code-ai-evals/); + assert.match(await warn(), /CODE_OF_CONDUCT\.md: still names the template's contact, opensource@github\.com/); + // Before the rename, the template's own owners and contact are expected. + assert.doesNotMatch(await warn({ name: "plugin-marketplace-starter", owner: { name: "GitHub" } }), /CODEOWNERS|CODE_OF_CONDUCT/); + // A comment that mentions the template's team isn't a rule. + await write(".github/CODEOWNERS", "# Replace @github/code-ai-evals with your own owners.\n* @acme/plugins\n"); + await write("CODE_OF_CONDUCT.md", "Report it to .\n"); + assert.doesNotMatch(await warn(), /CODEOWNERS|CODE_OF_CONDUCT/); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + test("verify warns about scaffold text in every scaffolded file and the template's introduction", async () => { const directory = await mkdtemp(join(tmpdir(), "starter-scaffold-")); const write = async (path, text) => { await mkdir(join(directory, path, ".."), { recursive: true }); await writeFile(join(directory, path), text); return join(directory, path); }; From 5e5ed02ca019aae9651d33e7543c51ab8fb7ead6 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Tue, 27 Oct 2026 14:06:40 -0700 Subject: [PATCH 2/6] Remove links to the GitHub Copilot Evals docs site The docs site isn't public, so a public repository can't link to it. Drop the aka.ms/evals/site links from the README, AGENTS.md, SUPPORT.md, the evals and remote plugin docs, .vally.yaml, and the remote eval plugin's catalog homepage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .vally.yaml | 2 +- AGENTS.md | 3 +-- README.md | 3 +-- SUPPORT.md | 6 ++---- docs/evals.md | 6 +++--- docs/remote-plugins.md | 5 ++--- marketplace.json | 3 +-- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.vally.yaml b/.vally.yaml index 998df46..4263e48 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -11,7 +11,7 @@ paths: evals: - evals # eval.yaml is this repository's convention; *.eval.yaml is the GitHub - # Copilot Evals convention (https://aka.ms/evals/site). Both work. + # Copilot Evals convention. Both work. evalFilenames: - eval.yaml - "*.eval.yaml" diff --git a/AGENTS.md b/AGENTS.md index cf7de81..dbfb9de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,8 +104,7 @@ Consult the relevant contract before changing package or catalog fields: - [Creating Copilot plugins](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/plugins-creating). - [Creating a team marketplace](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/plugins-marketplace). - [CLI plugin and catalog reference](https://docs.github.com/en/copilot/reference/copilot-cli-reference/cli-plugin-reference). -- [Vally eval schema](https://microsoft.github.io/vally/reference/eval-spec/) and - [GitHub Copilot Evals docs](https://aka.ms/evals/site). +- [Vally eval schema](https://microsoft.github.io/vally/reference/eval-spec/). The README's [references](README.md#references) cover skills, hooks, VS Code, and this repository's guides. \ No newline at end of file diff --git a/README.md b/README.md index 6895768..f07e0c6 100644 --- a/README.md +++ b/README.md @@ -358,5 +358,4 @@ to everyone who's contributed since. - [Hooks reference](https://docs.github.com/en/copilot/reference/hooks-reference) - [VS Code agent plugins](https://code.visualstudio.com/docs/agent-customization/agent-plugins) - [Agent Skills specification](https://agentskills.io/specification) -- [Vally](https://microsoft.github.io/vally/) and the - [GitHub Copilot Evals docs](https://aka.ms/evals/site) +- [Vally](https://microsoft.github.io/vally/) diff --git a/SUPPORT.md b/SUPPORT.md index 314faf1..9647988 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -14,10 +14,8 @@ Don't report security problems in public issues. Follow [SECURITY.md](SECURITY.md) instead. -For questions about GitHub Copilot Evals itself, see the -[GitHub Copilot Evals docs](https://aka.ms/evals/site). For problems with -Copilot CLI, the GitHub Copilot app, or the GitHub Copilot Chat VS Code -extension, rather than with this template, use +For problems with Copilot CLI, the GitHub Copilot app, or the GitHub Copilot +Chat VS Code extension, rather than with this template, use [GitHub Community discussions](https://github.com/orgs/community/discussions/categories/copilot-conversations) or [GitHub Support](https://support.github.com/). diff --git a/docs/evals.md b/docs/evals.md index 3111b73..49a8580 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -84,9 +84,9 @@ and check your billing dashboard after a run to learn what your suite costs. ## Write an eval 1. Create `evals//eval.yaml` with realistic tasks, expected outcomes, - and at least one negative or non-trigger case. The - [GitHub Copilot Evals](https://aka.ms/evals/site) convention, - `.eval.yaml`, works too; `evalFilenames` in `.vally.yaml` lists both. + and at least one negative or non-trigger case. The GitHub + Copilot Evals convention, `.eval.yaml`, works too; `evalFilenames` + in `.vally.yaml` lists both. Keep private production data out of fixtures and expected answers out of agent-visible inputs. Put grader programs and answer keys in `grading_environment`. diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index 4cf0242..fe5ba5a 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -36,9 +36,8 @@ Pick one route. Don't install the same plugin from both marketplaces. The pinned release is the built plugin. It provides the native `/eval` command (`/eval setup`, `create`, `review`, `improve`, `run`, and `compare`) in the CLI and Copilot App, a canvas, and six model-invocable skills. Running an eval -needs a compatible runner and its own approval, and it spends AI credits. See -the [GitHub Copilot Evals docs](https://aka.ms/evals/site) for VS Code and -Copilot App setup. This repository's eval tooling works without it. +needs a compatible runner and its own approval, and it spends AI credits. +This repository's eval tooling works without it. The `github/evals-dist` repository isn't public yet, so each installer needs a GitHub account with access. This catalog entry doesn't grant access or copy diff --git a/marketplace.json b/marketplace.json index 605d20f..7b6a4a6 100644 --- a/marketplace.json +++ b/marketplace.json @@ -48,8 +48,7 @@ "path": "packages/plugin" }, "version": "0.20.3", - "description": "GitHub Copilot Evals: set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running need separate approval.", - "homepage": "https://aka.ms/evals/site" + "description": "GitHub Copilot Evals: set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running need separate approval." } ] } From 49b252e4abeb1753788ffe7e7df9acd5d5731a61 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Tue, 27 Oct 2026 14:39:27 -0700 Subject: [PATCH 3/6] Pull the Evals plugin from github/evals again github/evals-dist is being renamed to github/evals, which will be the public distribution repo, so restore the original github-copilot catalog entry and its docs and tests instead of pointing at github/evals-dist. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 16 ++++++++-------- docs/remote-plugins.md | 27 ++++++++++++--------------- marketplace.json | 13 +++++++------ test/adoption.test.mjs | 2 +- test/catalog.test.mjs | 6 +++--- 5 files changed, 31 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index f07e0c6..2b6191f 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,10 @@ Two terms come up a lot: | [code-review](plugins/code-review/) | "Use adversarial-review to review this diff without changing it" | A skill that delegates to a read-only custom agent | | [json-configuration](plugins/json-configuration/) | 'Use json-configuration to validate {"port":8443} and look up /port' | A skill backed by a bundled local MCP server | - + The catalog also has an optional reference to the GitHub Copilot Evals plugin, which lives in another repository. See [Remote plugins](docs/remote-plugins.md). - + [Examples](docs/examples.md) explains what each example shows and which files it owns. @@ -104,12 +104,12 @@ You need Node.js 22.14 or newer. 4. Add your first plugin with `npm run new-plugin -- `. See [Add a plugin](#add-a-plugin). `npm run verify` passes even with no plugins at all. - -5. Decide whether to keep the remote `eval` plugin, GitHub Copilot Evals, in - the catalog. It installs from `github/evals-dist`, which isn't public yet; - see [Remote plugins](docs/remote-plugins.md). Remove it with - `npm run remove-plugin -- eval`. - + +5. Decide whether to keep the remote `github-copilot` plugin, GitHub Copilot + Evals, in the catalog. It installs from `github/evals`, which isn't public + yet; see [Remote plugins](docs/remote-plugins.md). Remove it with + `npm run remove-plugin -- github-copilot`. + 6. Replace what's meant for you before you ship: - Your own copyright line in [LICENSE](LICENSE), beside the template's. That one file covers the whole repository, including every plugin. See diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index fe5ba5a..94a6525 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -11,24 +11,21 @@ run upstream code, and the local evals don't cover remote plugins: `evals/` is for the plugins under `plugins/`, and the coverage check rejects asset IDs that aren't local. Evaluate an upstream plugin in its own repository. - + ## The GitHub Copilot Evals plugin - -[marketplace.json](../marketplace.json) includes an `eval` entry that points at -the published GitHub Copilot Evals plugin in `github/evals-dist`. It mirrors -the entry in upstream's own catalog: the `copilot-evals-v0.20.3` release, -pinned to the full SHA of its commit. +[marketplace.json](../marketplace.json) includes a `github-copilot` entry that +points at the published GitHub Copilot Evals plugin in `github/evals`. It's +the `copilot-evals-v0.16.0` release, pinned to the full SHA of its built +commit. After adding this marketplace, install it explicitly with -`copilot plugin install eval@MARKETPLACE`, where `MARKETPLACE` is the -`name` in `marketplace.json`. Or skip this catalog and install it from -upstream, which follows upstream's releases: +`copilot plugin install github-copilot@MARKETPLACE`, where `MARKETPLACE` is the +`name` in `marketplace.json`. Or skip this catalog and add upstream's own +marketplace, which follows upstream's releases: ```sh -copilot plugin marketplace add github/evals-dist -copilot plugin install eval@github-copilot-evals-dist +copilot plugin marketplace add github/evals ``` Pick one route. Don't install the same plugin from both marketplaces. @@ -39,11 +36,11 @@ and Copilot App, a canvas, and six model-invocable skills. Running an eval needs a compatible runner and its own approval, and it spends AI credits. This repository's eval tooling works without it. -The `github/evals-dist` repository isn't public yet, so each installer needs a +The `github/evals` repository isn't public yet, so each installer needs a GitHub account with access. This catalog entry doesn't grant access or copy upstream code. If your team doesn't have access, remove the entry with -`npm run remove-plugin -- eval`. - +`npm run remove-plugin -- github-copilot`. + ## Add a remote plugin diff --git a/marketplace.json b/marketplace.json index 7b6a4a6..5b3add7 100644 --- a/marketplace.json +++ b/marketplace.json @@ -39,16 +39,17 @@ "description": "Validate JSON and retrieve configuration values with a local MCP server." }, { - "name": "eval", + "name": "github-copilot", "source": { "source": "github", - "repo": "github/evals-dist", - "ref": "copilot-evals-v0.20.3", - "sha": "acd4b5029b0791fbe1c4b58e28fa94c580841d69", + "repo": "github/evals", + "ref": "copilot-evals-v0.16.0", + "sha": "dd4c5015b5e8fdfcbd17b261d996c85aa92fde5b", "path": "packages/plugin" }, - "version": "0.20.3", - "description": "GitHub Copilot Evals: set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running need separate approval." + "version": "0.16.0", + "description": "Configure native projects with /eval setup, create and save evals with /eval create, save distinct candidates with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running follow separate host approval policies.", + "homepage": "https://github.com/github/evals" } ] } diff --git a/test/adoption.test.mjs b/test/adoption.test.mjs index b72f627..9b67242 100644 --- a/test/adoption.test.mjs +++ b/test/adoption.test.mjs @@ -187,7 +187,7 @@ test("an empty or remote-only marketplace passes verify, and new plugins scaffol const setup = (await readFile(join(directory, "README.md"), "utf8")).match(/## Make it yours\n[\s\S]*?\n## /)?.[0]; if (setup) { if (present.length) assert.doesNotMatch(setup, /remove-example/); - assert.doesNotMatch(setup, /remove-plugin -- eval\b|evals-dist/); + assert.doesNotMatch(setup, /github-copilot/); const numbers = [...setup.matchAll(/^(\d+)\. /gm)].map((match) => Number(match[1])); assert.deepEqual(numbers, numbers.map((_, index) => index + 1)); } diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs index 7d07538..947decb 100644 --- a/test/catalog.test.mjs +++ b/test/catalog.test.mjs @@ -9,10 +9,10 @@ import { discoverAssets, checkCoverage, root } from "../scripts/validate-evals.m test("the catalog separates local plugins from pinned, opt-in remote references", async () => { const { catalog, localPlugins, remotePlugins } = await readCatalog(root); assert.equal(localPlugins.length + remotePlugins.length, catalog.plugins.length); - const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals-dist"); + const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals"); if (evals) { // The catalog name must match upstream's plugin.json name, not its marketplace name. - assert.equal(evals.name, "eval"); + assert.equal(evals.name, "github-copilot"); assert.equal(evals.source.path, "packages/plugin"); assert.deepEqual(remoteWarnings(evals), []); assert.equal(evals.version, evals.source.ref.replace(/^copilot-evals-v/, "")); @@ -37,7 +37,7 @@ test("an empty catalog is valid, and catalog errors are plain sentences", () => test("remote references must pin a full sha, and errors name the entry", () => { const sha = "a".repeat(40); for (const source of [ - { source: "github", repo: "example/evals", path: "packages/plugin", sha }, + { source: "github", repo: "github/evals", path: "packages/plugin", sha }, { source: "github", repo: "example/plugin", ref: "v1.0.0", sha }, { source: "github", repo: "example/plugin", sha }, ]) assert.doesNotThrow(() => validateRemoteSource(source, "fixture")); From b7ff751fcef0414d2a5d6ea837945096005aca2f Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Tue, 27 Oct 2026 15:05:28 -0700 Subject: [PATCH 4/6] Remove the not-public-yet disclaimers github/evals and github/evals-action open alongside this repository, so drop the "isn't public yet" wording and TODOs from the eval workflow, README, CI, remote plugin, and security docs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/evals.yml | 8 +++----- README.md | 11 +++++------ docs/ci.md | 11 +++-------- docs/remote-plugins.md | 6 ++---- docs/security.md | 2 +- 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 9972149..c900744 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -4,9 +4,9 @@ name: Behavioral evals # ENABLE_EVALS to true. Manual runs work either way. The repository variable # EVALS_RUNNER picks the job: unset or "npm" runs the pinned Vally CLI from # package.json, which works in any repository; "action" uses the GitHub Copilot -# Evals Action (github/evals-action), which isn't public yet. Any other value -# fails. Both eval jobs use the "evals" environment, so you can require a -# reviewer to approve each run before it gets a token. See docs/ci.md. +# Evals Action (github/evals-action). Any other value fails. Both eval jobs use +# the "evals" environment, so you can require a reviewer to approve each run +# before it gets a token. See docs/ci.md. on: pull_request: types: [opened, synchronize, reopened, ready_for_review] @@ -85,8 +85,6 @@ jobs: run: node "$COPILOT_CLI_PATH" --version env: COPILOT_CLI_PATH: ${{ github.workspace }}/scripts/eval-runtime.js - # TODO: github/evals-action isn't public yet. When it is, move this pin to - # the release that ships its third-party notices. - uses: github/evals-action@83fa12332c3e44ad899f347eaf153a0d9b6432dc # v0.1.0 env: # Vally's SDK executor launches this bridge, which runs the pinned full runtime. diff --git a/README.md b/README.md index 2b6191f..f8f2f06 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,8 @@ You need Node.js 22.14 or newer. at all. 5. Decide whether to keep the remote `github-copilot` plugin, GitHub Copilot - Evals, in the catalog. It installs from `github/evals`, which isn't public - yet; see [Remote plugins](docs/remote-plugins.md). Remove it with + Evals, in the catalog. It installs from `github/evals`; see + [Remote plugins](docs/remote-plugins.md). Remove it with `npm run remove-plugin -- github-copilot`. 6. Replace what's meant for you before you ship: @@ -264,10 +264,9 @@ pull requests until you set the repository variable `ENABLE_EVALS` to `true`, and you can always run it by hand. Each run can wait for a reviewer's approval through the `evals` environment. By default it runs the same evals with the pinned Vally CLI and the job's token, so it works in any repository, including -one on a personal account. Repositories that can use the GitHub Copilot Evals -Action, which isn't public yet, can set the repository variable `EVALS_RUNNER` -to `action` instead. On a personal account, usage is -billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers +one on a personal account. To use the GitHub Copilot Evals Action instead, set +the repository variable `EVALS_RUNNER` to `action`. On a personal account, +usage is billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers setup, personal accounts, billing, cost, and security. ## Repository layout diff --git a/docs/ci.md b/docs/ci.md index 5a2e5ab..0f7af50 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -40,8 +40,7 @@ organization: - `validate.yml` works as it is, and it's all you need to keep the marketplace consistent. It never calls a model. - To run behavioral evals in CI, leave the repository variable `EVALS_RUNNER` - unset or set it to `npm`. The `action` runner uses an Action that isn't - public yet, so it fails outside the GitHub organization. + unset or set it to `npm`, or set it to `action` to use the Evals Action. - Copilot usage from the job's token is [billed to your own Copilot plan](https://docs.github.com/en/copilot/concepts/agent-capabilities/agent-powered-workflows-and-operations/copilot-cli-in-github-actions). Each plan includes a monthly allowance of AI credits; see @@ -106,15 +105,11 @@ as drafts. | | `npm` (default) | `action` | |---|---|---| | Set `EVALS_RUNNER` to | nothing, or `npm` | `action` | -| Runs | `node scripts/evals.mjs eval --suite --require-pass` | The GitHub Copilot Evals Action, `github/evals-action` v0.1.0 | -| Who can use it | Any repository | Repositories in the GitHub organization, for now. The Action isn't public yet, so other organizations can't use it. | +| Runs | `node scripts/evals.mjs eval --suite --require-pass` | The GitHub Copilot Evals Action, [`github/evals-action`](https://github.com/github/evals-action) v0.1.0 | +| Who can use it | Any repository | Any repository | | Report | The job summary | Structured report and one sticky pull request comment | | Job token permissions | `contents: read`, `copilot-requests: write` | `contents: read`, `copilot-requests: write`, `pull-requests: write` | - - The two runners are separate jobs, because a job downloads every action it names before it starts, even ones in skipped steps. With the npm runner, the Evals Action is never downloaded. diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index 94a6525..939a052 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -36,10 +36,8 @@ and Copilot App, a canvas, and six model-invocable skills. Running an eval needs a compatible runner and its own approval, and it spends AI credits. This repository's eval tooling works without it. -The `github/evals` repository isn't public yet, so each installer needs a -GitHub account with access. This catalog entry doesn't grant access or copy -upstream code. If your team doesn't have access, remove the entry with -`npm run remove-plugin -- github-copilot`. +This catalog entry doesn't copy upstream code. If you don't want it, remove the +entry with `npm run remove-plugin -- github-copilot`. ## Add a remote plugin diff --git a/docs/security.md b/docs/security.md index 4d56716..6878e44 100644 --- a/docs/security.md +++ b/docs/security.md @@ -32,7 +32,7 @@ plugins. | Local eval results in `vally-results/`: results, a report, JUnit, and session evidence with transcripts | Your disk | Gitignored. Delete them when you're done. | | npm runner in CI: Vally's log and the report in the job summary | The workflow run | Anyone who can read the repository's Actions runs. Logs follow the repository's [retention setting](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization) (90 days by default). | | npm runner results artifact, which includes transcripts | A workflow artifact | Only when `EVALS_UPLOAD_RESULTS` is `true`. Kept for 7 days. | -| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. The Action isn't public yet, and this template doesn't document its internals. | +| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. This template doesn't document the Action's internals. | | Vally telemetry | Vally's telemetry service, unless turned off | `VALLY_TELEMETRY_OPTOUT=1` is set by `validate.yml`, the local launcher, and both eval jobs. The Action job also forwards it to Vally's run phase; whether the Action's lint and report phases honor it isn't verified. | | Copilot CLI telemetry | GitHub | Set by Copilot's own settings and policies, not by this template. | From 825b85e4b0f2fb816d3eb35fc30a94a6be06e702 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Tue, 27 Oct 2026 16:04:48 -0700 Subject: [PATCH 5/6] Limit the release prep to the OSPO Prepare Project checklist Revert changes no Prepare Project item requires: the npm default runner in evals.yml, the verify warnings for the template's code owners and conduct contact, the CODEOWNERS header rewording, the CLA note in CONTRIBUTING.md, the extra adopter steps in the README, the catalog homepage anchor, and link and wording edits for github/evals and github/evals-action, which open alongside this repository. Use the OSPO SECURITY.md, SUPPORT.md, CODE_OF_CONDUCT.md and CONTRIBUTING.md template text without extras. Keep only the minimum edits that drop the not-public-yet disclaimers and the internal docs links. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/CODEOWNERS | 9 ++---- .github/workflows/evals.yml | 20 ++++++------- .vally.yaml | 4 +-- AGENTS.md | 3 +- CODE_OF_CONDUCT.md | 1 - CONTRIBUTING.md | 13 ++++----- README.md | 24 +++++++--------- SECURITY.md | 57 ++++++++++++++++--------------------- SUPPORT.md | 22 +++----------- docs/ci.md | 33 +++++++++++---------- docs/evals.md | 10 +++---- docs/remote-plugins.md | 12 ++++---- marketplace.json | 2 +- scripts/catalog.mjs | 2 +- scripts/validate.mjs | 10 ------- test/eval-setup.test.mjs | 6 ++-- test/marketplace.test.mjs | 21 -------------- 17 files changed, 94 insertions(+), 155 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 892a66d..368d48b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,11 +1,8 @@ -# Code owners review changes to the paths below. The template's own owners are -# its maintainers, @github/code-ai-evals. Replace them with real GitHub users or -# teams before you ship, then require code owner review in a branch protection -# rule or ruleset. This file alone doesn't enforce review. +# Code owners review changes to the paths below. Replace these examples with +# real GitHub users or teams before you ship, then require code owner review in +# a branch protection rule or ruleset. This file alone doesn't enforce review. # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # -# Examples for a marketplace of your own: -# # * @your-org/ai-customizations # /.github/workflows/ @your-org/ai-customizations-admins # /.github/copilot/settings.json @your-org/ai-customizations-admins diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index c900744..8c3f7b6 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -2,11 +2,10 @@ name: Behavioral evals # Off by default, because evals run real Copilot sessions and spend AI credits. # Pull requests run evals only after you set the repository variable # ENABLE_EVALS to true. Manual runs work either way. The repository variable -# EVALS_RUNNER picks the job: unset or "npm" runs the pinned Vally CLI from -# package.json, which works in any repository; "action" uses the GitHub Copilot -# Evals Action (github/evals-action). Any other value fails. Both eval jobs use -# the "evals" environment, so you can require a reviewer to approve each run -# before it gets a token. See docs/ci.md. +# EVALS_RUNNER picks the job: unset or "action" uses github/evals-action; +# "npm" runs the pinned Vally CLI from package.json. Any other value fails. Both eval +# jobs use the "evals" environment, so you can require a reviewer to approve +# each run before it gets a token. See docs/ci.md. on: pull_request: types: [opened, synchronize, reopened, ready_for_review] @@ -41,13 +40,13 @@ jobs: env: EVALS_RUNNER: ${{ vars.EVALS_RUNNER }} - # Runner "action", opt-in. A separate job, because a job downloads every - # action it names before it starts, even ones in skipped steps. + # Runner "action". A separate job, because a job downloads every action it + # names before it starts, even ones in skipped steps. evals: # Manual runs always run. Pull requests run only when ENABLE_EVALS is true, # the branch is in this repository (never a fork), and the PR isn't a draft. if: >- - vars.EVALS_RUNNER == 'action' && (github.event_name == 'workflow_dispatch' || + (vars.EVALS_RUNNER == '' || vars.EVALS_RUNNER == 'action') && (github.event_name == 'workflow_dispatch' || (vars.ENABLE_EVALS == 'true' && github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft)) @@ -107,11 +106,10 @@ jobs: max-retries: 1 upload-transcripts: false - # Runner "npm", the default: the same launcher contributors use locally, with - # the job token. + # Runner "npm": the same launcher contributors use locally, with the job token. evals-npm: if: >- - (vars.EVALS_RUNNER == '' || vars.EVALS_RUNNER == 'npm') && (github.event_name == 'workflow_dispatch' || + vars.EVALS_RUNNER == 'npm' && (github.event_name == 'workflow_dispatch' || (vars.ENABLE_EVALS == 'true' && github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft)) diff --git a/.vally.yaml b/.vally.yaml index 4263e48..33de6d7 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -10,8 +10,8 @@ paths: - plugins/json-configuration/skills evals: - evals - # eval.yaml is this repository's convention; *.eval.yaml is the GitHub - # Copilot Evals convention. Both work. + # eval.yaml is this repository's convention; *.eval.yaml is the upstream + # github/evals convention. Both work. evalFilenames: - eval.yaml - "*.eval.yaml" diff --git a/AGENTS.md b/AGENTS.md index dbfb9de..9465fdb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,7 +104,8 @@ Consult the relevant contract before changing package or catalog fields: - [Creating Copilot plugins](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/plugins-creating). - [Creating a team marketplace](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/plugins-marketplace). - [CLI plugin and catalog reference](https://docs.github.com/en/copilot/reference/copilot-cli-reference/cli-plugin-reference). -- [Vally eval schema](https://microsoft.github.io/vally/reference/eval-spec/). +- [Vally eval schema](https://microsoft.github.io/vally/reference/eval-spec/) and + [Evals Action](https://github.com/github/evals-action). The README's [references](README.md#references) cover skills, hooks, VS Code, and this repository's guides. \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index baf4348..6dc4b12 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -54,7 +54,6 @@ further defined and clarified by project maintainers. ## Enforcement - Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13324cc..d8ff35c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,14 +6,11 @@ explains how everything fits together. Contributions to this project are [released](https://docs.github.com/en/site-policy/github-terms/github-terms-of-service#6-contributions-under-repository-license) -to the public under the [project's open source license](LICENSE). This project -is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By -participating in it, you agree to abide by its terms. - - -You may need to agree to a contributor agreement before your pull request can -be merged. - +to the public under the [project's open source license](LICENSE). + +Please note that this project is released with a +[Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this +project you agree to abide by its terms. ## Before you start diff --git a/README.md b/README.md index f8f2f06..c2d17e8 100644 --- a/README.md +++ b/README.md @@ -105,8 +105,7 @@ You need Node.js 22.14 or newer. [Add a plugin](#add-a-plugin). `npm run verify` passes even with no plugins at all. -5. Decide whether to keep the remote `github-copilot` plugin, GitHub Copilot - Evals, in the catalog. It installs from `github/evals`; see +5. Decide whether to keep the remote `github-copilot` plugin in the catalog. See [Remote plugins](docs/remote-plugins.md). Remove it with `npm run remove-plugin -- github-copilot`. @@ -115,11 +114,9 @@ You need Node.js 22.14 or newer. one file covers the whole repository, including every plugin. See [Licensing](#licensing). - The title and introduction of this README. - - Real owners in [.github/CODEOWNERS](.github/CODEOWNERS), in place of the - template's maintainers. On a personal account, that can be just you. - - Your reporting channel in [SECURITY.md](SECURITY.md), your enforcement - contact in [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md), and how people get - help in [SUPPORT.md](SUPPORT.md). + - Real owners in [.github/CODEOWNERS](.github/CODEOWNERS). On a personal + account, that can be just you. + - Your reporting channel in [SECURITY.md](SECURITY.md). - Your process in [CONTRIBUTING.md](CONTRIBUTING.md), the [pull request template](.github/PULL_REQUEST_TEMPLATE.md), and the [new plugin issue form](.github/ISSUE_TEMPLATE/new-plugin.yml). @@ -262,11 +259,11 @@ marketplace consistent. [evals.yml](.github/workflows/evals.yml) runs behavioral evals. It's off for pull requests until you set the repository variable `ENABLE_EVALS` to `true`, and you can always run it by hand. Each run can wait for a reviewer's approval -through the `evals` environment. By default it runs the same evals with the -pinned Vally CLI and the job's token, so it works in any repository, including -one on a personal account. To use the GitHub Copilot Evals Action instead, set -the repository variable `EVALS_RUNNER` to `action`. On a personal account, -usage is billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers +through the `evals` environment. By default it uses +[`github/evals-action`](https://github.com/github/evals-action). Set the +repository variable `EVALS_RUNNER` to `npm` to run +the same evals with the pinned Vally CLI. On a personal account, usage is +billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers setup, personal accounts, billing, cost, and security. ## Repository layout @@ -357,4 +354,5 @@ to everyone who's contributed since. - [Hooks reference](https://docs.github.com/en/copilot/reference/hooks-reference) - [VS Code agent plugins](https://code.visualstudio.com/docs/agent-customization/agent-plugins) - [Agent Skills specification](https://agentskills.io/specification) -- [Vally](https://microsoft.github.io/vally/) +- [Vally](https://microsoft.github.io/vally/) and the + [Evals Action](https://github.com/github/evals-action) diff --git a/SECURITY.md b/SECURITY.md index e53a14b..68e5f93 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,40 +5,33 @@ Don't report security problems in public issues, discussions, or pull requests. -### In this template - -Thanks for helping make GitHub safe for everyone. GitHub takes the security of -its software products and services seriously, including all of the open source -code repositories managed through its GitHub organizations, such as -[GitHub](https://github.com/GitHub). - -If you believe you've found a security vulnerability in this template, -`github/plugin-marketplace-starter`, or in any other GitHub-owned repository, -report it to GitHub through coordinated disclosure. Send an email to -opensource-security[@]github.com. - -Include as much of the following as you can. It helps GitHub understand and -triage your report more quickly. - -- The type of issue, such as command injection, path traversal, or cross-site - scripting -- Full paths of the source files related to the issue -- The location of the affected source code: a tag, branch, commit, or direct - URL -- Any special configuration needed to reproduce the issue -- Step-by-step instructions to reproduce the issue -- Proof-of-concept or exploit code, if possible -- The impact of the issue, including how an attacker might exploit it - -Open source repositories are -[outside the scope of GitHub's bug bounty program](https://bounty.github.com/index.html#scope), -so they aren't eligible for bounty rewards, but GitHub will make sure your -finding reaches the right maintainers. See -[GitHub's Safe Harbor Policy](https://docs.github.com/en/site-policy/security-policies/github-bug-bounty-program-legal-safe-harbor#1-safe-harbor-terms). - -### In a marketplace made from this template +Thanks for helping make GitHub safe for everyone. +GitHub takes the security of our software products and services seriously, including all of the open source code repositories managed through our GitHub organizations, such as [GitHub](https://github.com/GitHub). + +Even though [open source repositories are outside of the scope of our bug bounty program](https://bounty.github.com/index.html#scope) and therefore not eligible for bounty rewards, we will ensure that your finding gets passed along to the appropriate maintainers for remediation. + +If you believe you have found a security vulnerability in any GitHub-owned repository, please report it to us through coordinated disclosure. + +**Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.** + +Instead, please send an email to opensource-security[@]github.com. + +Please include as much of the information listed below as you can to help us better understand and resolve the issue: + + * The type of issue (e.g., buffer overflow, SQL injection, or cross-site scripting) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +See [GitHub's Safe Harbor Policy](https://docs.github.com/en/site-policy/security-policies/github-bug-bounty-program-legal-safe-harbor#1-safe-harbor-terms) + Use this repository's **Security** tab to [report a vulnerability privately](https://docs.github.com/en/code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository), diff --git a/SUPPORT.md b/SUPPORT.md index 9647988..73e153a 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -2,28 +2,14 @@ ## How to file issues and get help -This project uses GitHub issues to track bugs and feature requests. Search the -existing issues before you file a new one, to avoid duplicates. For a new bug -or feature request, file a new issue. To propose a plugin, use the -**New plugin** issue form. +This project uses GitHub issues to track bugs and feature requests. Please search the existing issues before filing new issues to avoid duplicates. For new issues, file your bug or feature request as a new issue. -For help or questions about using this marketplace, open an issue or ask the -maintainers listed in [.github/CODEOWNERS](.github/CODEOWNERS). - -Don't report security problems in public issues. Follow -[SECURITY.md](SECURITY.md) instead. +For help or questions about using this project, please open an issue. -For problems with Copilot CLI, the GitHub Copilot app, or the GitHub Copilot -Chat VS Code extension, rather than with this template, use -[GitHub Community discussions](https://github.com/orgs/community/discussions/categories/copilot-conversations) -or [GitHub Support](https://support.github.com/). - -This template is under active development and maintained by GitHub staff and -the community. We'll do our best to respond to support, feature requests, and -community questions in a timely manner. +The plugin marketplace starter is under active development and maintained by GitHub staff and the community. We will do our best to respond to support, feature requests, and community questions in a timely manner. -## Support policy +## GitHub Support Policy Support for this project is limited to the resources listed above. diff --git a/docs/ci.md b/docs/ci.md index 0f7af50..4b6a814 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -39,8 +39,8 @@ organization: - `validate.yml` works as it is, and it's all you need to keep the marketplace consistent. It never calls a model. -- To run behavioral evals in CI, leave the repository variable `EVALS_RUNNER` - unset or set it to `npm`, or set it to `action` to use the Evals Action. +- To run behavioral evals in CI, set the repository variable `EVALS_RUNNER` to + `npm`. - Copilot usage from the job's token is [billed to your own Copilot plan](https://docs.github.com/en/copilot/concepts/agent-capabilities/agent-powered-workflows-and-operations/copilot-cli-in-github-actions). Each plan includes a monthly allowance of AI credits; see @@ -68,10 +68,9 @@ turn it on. ### Turn it on 1. Check the [requirements](#requirements) below. -2. Pick a runner (see the next section). The npm runner is the default. To use - the Evals Action instead, create the repository variable `EVALS_RUNNER` - with the value `action`. Any value other than `action`, `npm`, or nothing - fails the workflow instead of guessing. +2. Pick a runner (see the next section). If you use the npm runner, create the + repository variable `EVALS_RUNNER` with the value `npm`. Any value other + than `action`, `npm`, or nothing fails the workflow instead of guessing. 3. Decide who approves runs. Both eval jobs use an environment named `evals`, which GitHub creates the first time a job uses it. To make a person approve each run before it gets a token, add required reviewers to it in @@ -96,23 +95,23 @@ Pull request runs start only when all of these hold: - It touches plugins, evals, schemas, scripts, `.vally.yaml`, the package files, or the workflow itself. -Runs don't cancel each other, which the Evals Action requires, so each push to -an open pull request queues one more run. Keep work-in-progress pull requests -as drafts. +Runs don't cancel each other, because the Evals Action requires that, so each +push to an open pull request queues one more run. Keep work-in-progress pull +requests as drafts. ### Pick a runner -| | `npm` (default) | `action` | +| | `action` (default) | `npm` | |---|---|---| -| Set `EVALS_RUNNER` to | nothing, or `npm` | `action` | -| Runs | `node scripts/evals.mjs eval --suite --require-pass` | The GitHub Copilot Evals Action, [`github/evals-action`](https://github.com/github/evals-action) v0.1.0 | +| Set `EVALS_RUNNER` to | nothing, or `action` | `npm` | +| Runs | [`github/evals-action`](https://github.com/github/evals-action) v0.1.0 | `node scripts/evals.mjs eval --suite --require-pass` | | Who can use it | Any repository | Any repository | -| Report | The job summary | Structured report and one sticky pull request comment | -| Job token permissions | `contents: read`, `copilot-requests: write` | `contents: read`, `copilot-requests: write`, `pull-requests: write` | +| Report | Structured report and one sticky pull request comment | The job summary | +| Job token permissions | `contents: read`, `copilot-requests: write`, `pull-requests: write` | `contents: read`, `copilot-requests: write` | The two runners are separate jobs, because a job downloads every action it -names before it starts, even ones in skipped steps. With the npm runner, the -Evals Action is never downloaded. +names before it starts, even ones in skipped steps. With `EVALS_RUNNER=npm`, the +Action is never downloaded. Both fail the job on a failed verdict, an incomplete run, or an infrastructure error, unlike the report-only default of `npm run eval:report` locally. Both use @@ -175,7 +174,7 @@ repository variable `EVALS_UPLOAD_RESULTS` to `true`. Both jobs use `ubuntu-latest`. - **Actions policy.** If your organization restricts which actions can run, allow the actions the workflow pins: `actions/checkout`, `actions/setup-node`, - `actions/upload-artifact`, and, for the `action` runner, `github/evals-action`. + `actions/upload-artifact`, and, for the default runner, `github/evals-action`. ## Cost diff --git a/docs/evals.md b/docs/evals.md index 49a8580..2fe5800 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -84,9 +84,9 @@ and check your billing dashboard after a run to learn what your suite costs. ## Write an eval 1. Create `evals//eval.yaml` with realistic tasks, expected outcomes, - and at least one negative or non-trigger case. The GitHub - Copilot Evals convention, `.eval.yaml`, works too; `evalFilenames` - in `.vally.yaml` lists both. + and at least one negative or non-trigger case. The upstream + [github/evals](https://github.com/github/evals) convention, + `.eval.yaml`, works too; `evalFilenames` in `.vally.yaml` lists both. Keep private production data out of fixtures and expected answers out of agent-visible inputs. Put grader programs and answer keys in `grading_environment`. @@ -218,8 +218,8 @@ them. Test them by hand in the hosts you support. ## Suites Suites are conventions, not fixed names. This repository's `smoke` suite -selects cases tagged `suite: smoke`, and `full` runs every spec. GitHub Copilot -Evals examples filter on `lane: ci` instead. A `tags.suite` value must +selects cases tagged `suite: smoke`, and `full` runs every spec. Upstream +github/evals examples filter on `lane: ci` instead. A `tags.suite` value must match a suite filter in `.vally.yaml`, and every asset needs a case in `smoke` because CI runs it by default. diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index 939a052..69558b0 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -15,17 +15,19 @@ aren't local. Evaluate an upstream plugin in its own repository. ## The GitHub Copilot Evals plugin [marketplace.json](../marketplace.json) includes a `github-copilot` entry that -points at the published GitHub Copilot Evals plugin in `github/evals`. It's -the `copilot-evals-v0.16.0` release, pinned to the full SHA of its built -commit. +points at the published GitHub Copilot Evals plugin in +[github/evals](https://github.com/github/evals). It mirrors the stable entry in +upstream's own catalog: the `copilot-evals-v0.16.0` release, pinned to the full +SHA of its built commit. After adding this marketplace, install it explicitly with `copilot plugin install github-copilot@MARKETPLACE`, where `MARKETPLACE` is the -`name` in `marketplace.json`. Or skip this catalog and add upstream's own -marketplace, which follows upstream's releases: +`name` in `marketplace.json`. Or skip this catalog and install it from +upstream, which follows upstream's reviewed stable releases: ```sh copilot plugin marketplace add github/evals +copilot plugin install github-copilot@evals ``` Pick one route. Don't install the same plugin from both marketplaces. diff --git a/marketplace.json b/marketplace.json index 5b3add7..754e01e 100644 --- a/marketplace.json +++ b/marketplace.json @@ -49,7 +49,7 @@ }, "version": "0.16.0", "description": "Configure native projects with /eval setup, create and save evals with /eval create, save distinct candidates with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running follow separate host approval policies.", - "homepage": "https://github.com/github/evals" + "homepage": "https://github.com/github/evals#installed-skills" } ] } diff --git a/scripts/catalog.mjs b/scripts/catalog.mjs index 05e4335..fe3a4e3 100644 --- a/scripts/catalog.mjs +++ b/scripts/catalog.mjs @@ -48,7 +48,7 @@ export function validateRemoteSource(source, name = "remote plugin") { assert.ok(!unknown.length, fail(`unknown source field ${unknown.join(", ")}. Use only source, repo, path, ref, and sha.`)); assert.equal(source.source, "github", fail('only "source": "github" remote references are supported.')); assert.ok(typeof source.repo === "string" && /^[A-Za-z0-9][A-Za-z0-9-]*\/(?!\.{1,2}$)[A-Za-z0-9_.-]+$/.test(source.repo), - fail('"repo" must be OWNER/REPO, such as "my-org/plugins", not a URL.')); + fail('"repo" must be OWNER/REPO, such as "github/evals", not a URL.')); if (source.path !== undefined) { assert.ok(relativePath(source.path), fail(`"path" must be a directory relative to the repository root, such as "packages/plugin", without a leading "./" or "/", "..", or backslashes. Omit "path" when plugin.json is at the repository root.`)); } diff --git a/scripts/validate.mjs b/scripts/validate.mjs index aa6a912..13fe62f 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -180,10 +180,6 @@ export const scaffoldText = Object.freeze([ // repository, plugins included. Its MIT notice stays in anything you keep; see // Licensing in README.md. export const templateLicenseHolder = "GitHub, Inc."; -// The template's own maintainers in .github/CODEOWNERS and its conduct contact -// in CODE_OF_CONDUCT.md. A marketplace made from the template names its own. -export const templateCodeOwners = "@github/code-ai-evals"; -export const templateConductContact = "opensource@github.com"; const read = async (path) => { try { return await readFile(path, "utf8"); } catch (error) { if (error.code === "ENOENT") return undefined; throw error; } }; // Every holder named on a "Copyright" line in a license file. @@ -239,12 +235,6 @@ async function adoptionWarnings(directory, catalog, allFiles) { if (owners !== undefined && !owners.split(/\r?\n/).some((line) => line.trim() && !line.trim().startsWith("#"))) { warnings.push(".github/CODEOWNERS: every rule is commented out. Name real owners, then require their review in a ruleset."); } - if (owners?.split(/\r?\n/).some((line) => !line.trim().startsWith("#") && line.split(/\s+/).includes(templateCodeOwners))) { - warnings.push(`.github/CODEOWNERS: still names the template's maintainers, ${templateCodeOwners}. Name your own owners, then require their review in a ruleset.`); - } - if ((await read(resolve(directory, "CODE_OF_CONDUCT.md")))?.includes(templateConductContact)) { - warnings.push(`CODE_OF_CONDUCT.md: still names the template's contact, ${templateConductContact}. Replace it with your own enforcement contact.`); - } return warnings; } diff --git a/test/eval-setup.test.mjs b/test/eval-setup.test.mjs index 280b7df..66f9b31 100644 --- a/test/eval-setup.test.mjs +++ b/test/eval-setup.test.mjs @@ -283,8 +283,8 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals } // Only one runner job can match a given EVALS_RUNNER. if (evals.jobs.evals && evals.jobs["evals-npm"]) { - assert.match(evals.jobs.evals.if, /^vars\.EVALS_RUNNER == 'action' &&/); - assert.match(evals.jobs["evals-npm"].if, /^\(vars\.EVALS_RUNNER == '' \|\| vars\.EVALS_RUNNER == 'npm'\) &&/); + assert.match(evals.jobs.evals.if, /^\(vars\.EVALS_RUNNER == '' \|\| vars\.EVALS_RUNNER == 'action'\) &&/); + assert.match(evals.jobs["evals-npm"].if, /^vars\.EVALS_RUNNER == 'npm' &&/); } const job = evals.jobs.evals; if (job) { @@ -300,7 +300,7 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals } const npmJob = evals.jobs["evals-npm"]; if (npmJob) { - assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the Evals Action."); + assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the Action."); const runStep = npmJob.steps.find((step) => /scripts\/evals\.mjs eval\b/.test(step.run ?? "")); assert.ok(runStep.run.includes("--require-pass")); const report = npmJob.steps.find((step) => /eval-report\.mjs/.test(step.run ?? "")); diff --git a/test/marketplace.test.mjs b/test/marketplace.test.mjs index 384dc67..f1c12ea 100644 --- a/test/marketplace.test.mjs +++ b/test/marketplace.test.mjs @@ -144,27 +144,6 @@ test("the license warning keeps the template's MIT notice and asks only for your } }); -test("after a rename, verify warns about the template's code owners and conduct contact", async () => { - const directory = await mkdtemp(join(tmpdir(), "starter-owners-")); - const write = async (path, text) => { await mkdir(join(directory, path, ".."), { recursive: true }); await writeFile(join(directory, path), text); return join(directory, path); }; - const warn = async (catalog = { name: "ours", owner: { name: "Acme" } }) => (await staleReferenceWarnings(directory, catalog, [])).join("\n"); - try { - await write("README.md", "# Ours\n"); - await write(".github/CODEOWNERS", "# Replace @github/code-ai-evals with your own owners.\n* @github/code-ai-evals\n"); - await write("CODE_OF_CONDUCT.md", "Report it to .\n"); - assert.match(await warn(), /\.github\/CODEOWNERS: still names the template's maintainers, @github\/code-ai-evals/); - assert.match(await warn(), /CODE_OF_CONDUCT\.md: still names the template's contact, opensource@github\.com/); - // Before the rename, the template's own owners and contact are expected. - assert.doesNotMatch(await warn({ name: "plugin-marketplace-starter", owner: { name: "GitHub" } }), /CODEOWNERS|CODE_OF_CONDUCT/); - // A comment that mentions the template's team isn't a rule. - await write(".github/CODEOWNERS", "# Replace @github/code-ai-evals with your own owners.\n* @acme/plugins\n"); - await write("CODE_OF_CONDUCT.md", "Report it to .\n"); - assert.doesNotMatch(await warn(), /CODEOWNERS|CODE_OF_CONDUCT/); - } finally { - await rm(directory, { recursive: true, force: true }); - } -}); - test("verify warns about scaffold text in every scaffolded file and the template's introduction", async () => { const directory = await mkdtemp(join(tmpdir(), "starter-scaffold-")); const write = async (path, text) => { await mkdir(join(directory, path, ".."), { recursive: true }); await writeFile(join(directory, path), text); return join(directory, path); }; From 29118904796f15868eeb67ce9b30912a4b50a109 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Tue, 27 Oct 2026 16:22:28 -0700 Subject: [PATCH 6/6] Re-pin the GitHub Copilot Evals plugin to github/evals v0.20.3 github/evals was renamed, and the old pin (copilot-evals-v0.16.0 at dd4c501) isn't a commit in the new github/evals, so npm run check-remotes failed. Pin the latest release, copilot-evals-v0.20.3 at acd4b50. Its plugin.json is named "eval", and check-remotes requires the catalog entry name to match, so rename the entry from github-copilot to eval and update the markers, remove-plugin and install commands, docs, and tests that use the old name. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 12 ++++++------ docs/remote-plugins.md | 14 +++++++------- marketplace.json | 8 ++++---- test/adoption.test.mjs | 2 +- test/catalog.test.mjs | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c2d17e8..248b0ee 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,10 @@ Two terms come up a lot: | [code-review](plugins/code-review/) | "Use adversarial-review to review this diff without changing it" | A skill that delegates to a read-only custom agent | | [json-configuration](plugins/json-configuration/) | 'Use json-configuration to validate {"port":8443} and look up /port' | A skill backed by a bundled local MCP server | - + The catalog also has an optional reference to the GitHub Copilot Evals plugin, which lives in another repository. See [Remote plugins](docs/remote-plugins.md). - + [Examples](docs/examples.md) explains what each example shows and which files it owns. @@ -104,11 +104,11 @@ You need Node.js 22.14 or newer. 4. Add your first plugin with `npm run new-plugin -- `. See [Add a plugin](#add-a-plugin). `npm run verify` passes even with no plugins at all. - -5. Decide whether to keep the remote `github-copilot` plugin in the catalog. See + +5. Decide whether to keep the remote `eval` plugin in the catalog. See [Remote plugins](docs/remote-plugins.md). Remove it with - `npm run remove-plugin -- github-copilot`. - + `npm run remove-plugin -- eval`. + 6. Replace what's meant for you before you ship: - Your own copyright line in [LICENSE](LICENSE), beside the template's. That one file covers the whole repository, including every plugin. See diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index 69558b0..417279f 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -11,23 +11,23 @@ run upstream code, and the local evals don't cover remote plugins: `evals/` is for the plugins under `plugins/`, and the coverage check rejects asset IDs that aren't local. Evaluate an upstream plugin in its own repository. - + ## The GitHub Copilot Evals plugin -[marketplace.json](../marketplace.json) includes a `github-copilot` entry that +[marketplace.json](../marketplace.json) includes an `eval` entry that points at the published GitHub Copilot Evals plugin in [github/evals](https://github.com/github/evals). It mirrors the stable entry in -upstream's own catalog: the `copilot-evals-v0.16.0` release, pinned to the full +upstream's own catalog: the `copilot-evals-v0.20.3` release, pinned to the full SHA of its built commit. After adding this marketplace, install it explicitly with -`copilot plugin install github-copilot@MARKETPLACE`, where `MARKETPLACE` is the +`copilot plugin install eval@MARKETPLACE`, where `MARKETPLACE` is the `name` in `marketplace.json`. Or skip this catalog and install it from upstream, which follows upstream's reviewed stable releases: ```sh copilot plugin marketplace add github/evals -copilot plugin install github-copilot@evals +copilot plugin install eval@github-copilot-evals-dist ``` Pick one route. Don't install the same plugin from both marketplaces. @@ -39,8 +39,8 @@ needs a compatible runner and its own approval, and it spends AI credits. This repository's eval tooling works without it. This catalog entry doesn't copy upstream code. If you don't want it, remove the -entry with `npm run remove-plugin -- github-copilot`. - +entry with `npm run remove-plugin -- eval`. + ## Add a remote plugin diff --git a/marketplace.json b/marketplace.json index 754e01e..60fe4a5 100644 --- a/marketplace.json +++ b/marketplace.json @@ -39,15 +39,15 @@ "description": "Validate JSON and retrieve configuration values with a local MCP server." }, { - "name": "github-copilot", + "name": "eval", "source": { "source": "github", "repo": "github/evals", - "ref": "copilot-evals-v0.16.0", - "sha": "dd4c5015b5e8fdfcbd17b261d996c85aa92fde5b", + "ref": "copilot-evals-v0.20.3", + "sha": "acd4b5029b0791fbe1c4b58e28fa94c580841d69", "path": "packages/plugin" }, - "version": "0.16.0", + "version": "0.20.3", "description": "Configure native projects with /eval setup, create and save evals with /eval create, save distinct candidates with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running follow separate host approval policies.", "homepage": "https://github.com/github/evals#installed-skills" } diff --git a/test/adoption.test.mjs b/test/adoption.test.mjs index 9b67242..7baf7b2 100644 --- a/test/adoption.test.mjs +++ b/test/adoption.test.mjs @@ -187,7 +187,7 @@ test("an empty or remote-only marketplace passes verify, and new plugins scaffol const setup = (await readFile(join(directory, "README.md"), "utf8")).match(/## Make it yours\n[\s\S]*?\n## /)?.[0]; if (setup) { if (present.length) assert.doesNotMatch(setup, /remove-example/); - assert.doesNotMatch(setup, /github-copilot/); + assert.doesNotMatch(setup, /plugin:eval\b|remove-plugin -- eval\b/); const numbers = [...setup.matchAll(/^(\d+)\. /gm)].map((match) => Number(match[1])); assert.deepEqual(numbers, numbers.map((_, index) => index + 1)); } diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs index 947decb..7f0209c 100644 --- a/test/catalog.test.mjs +++ b/test/catalog.test.mjs @@ -12,7 +12,7 @@ test("the catalog separates local plugins from pinned, opt-in remote references" const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals"); if (evals) { // The catalog name must match upstream's plugin.json name, not its marketplace name. - assert.equal(evals.name, "github-copilot"); + assert.equal(evals.name, "eval"); assert.equal(evals.source.path, "packages/plugin"); assert.deepEqual(remoteWarnings(evals), []); assert.equal(evals.version, evals.source.ref.replace(/^copilot-evals-v/, ""));