diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c730935..368d48b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,3 +10,6 @@ # /evals/ @your-org/ai-customizations # /marketplace.json @your-org/ai-customizations-admins # /plugins// @your-org/ai-customizations-admins + +# This repository is maintained by: +* @github/code-ai-evals diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index e306f9a..8c3f7b6 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -2,9 +2,8 @@ name: Behavioral evals # Off by default, because evals run real Copilot sessions and spend AI credits. # Pull requests run evals only after you set the repository variable # ENABLE_EVALS to true. Manual runs work either way. The repository variable -# EVALS_RUNNER picks the job: unset or "action" uses the private -# github/evals-action; "npm" runs the pinned Vally CLI from package.json, for -# organizations without access to that Action. Any other value fails. Both eval +# EVALS_RUNNER picks the job: unset or "action" uses github/evals-action; +# "npm" runs the pinned Vally CLI from package.json. Any other value fails. Both eval # jobs use the "evals" environment, so you can require a reviewer to approve # each run before it gets a token. See docs/ci.md. on: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..6dc4b12 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db76b6a..d8ff35c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,14 @@ Thanks for improving this marketplace. This guide is the checklist for people; [AGENTS.md](AGENTS.md) adds rules for coding agents, and the [README](README.md) explains how everything fits together. +Contributions to this project are +[released](https://docs.github.com/en/site-policy/github-terms/github-terms-of-service#6-contributions-under-repository-license) +to the public under the [project's open source license](LICENSE). + +Please note that this project is released with a +[Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this +project you agree to abide by its terms. + ## Before you start - Install Node.js 22.14 or newer and run `npm ci`. diff --git a/README.md b/README.md index ddaf078..248b0ee 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,11 @@ A GitHub template for building, evaluating, and sharing AI customizations works for a team in an organization and for one person with a personal account. + +**Status:** active development. GitHub maintains this template; see +[Maintainers](#maintainers) and [Support](#support). + + It gives you: @@ -38,15 +43,41 @@ Two terms come up a lot: | [code-review](plugins/code-review/) | "Use adversarial-review to review this diff without changing it" | A skill that delegates to a read-only custom agent | | [json-configuration](plugins/json-configuration/) | 'Use json-configuration to validate {"port":8443} and look up /port' | A skill backed by a bundled local MCP server | - + The catalog also has an optional reference to the GitHub Copilot Evals plugin, which lives in another repository. See [Remote plugins](docs/remote-plugins.md). - + [Examples](docs/examples.md) explains what each example shows and which files it owns. + +## Background + +Most shared Copilot customizations ship as loose files, without a catalog, +checks, or evals, because teams have to piece that setup together from docs. +This template puts the whole setup in one repository, so a new skill, agent, +hook, or MCP server arrives with a catalog entry, a model-free check, and an +eval that shows it works. + +Planned work and known gaps are in the +[open issues](https://github.com/github/plugin-marketplace-starter/issues). +Contributions are welcome, from fixes to the scripts and docs to better +examples and new checks. Read [CONTRIBUTING.md](CONTRIBUTING.md) first. + + +## Requirements + +- To install and use the plugins: a current Copilot CLI, the GitHub Copilot + app, or VS Code with agent plugins turned on, and a GitHub Copilot plan. See + [Install plugins](#install-plugins). +- To build, check, and change the marketplace: Node.js 22.14 or newer and npm. + `npm ci` installs everything else, including the pinned Vally CLI and Copilot + runtime that evals use. +- To run behavioral evals: a Copilot plan or organization policy that allows + the usage, since every run spends AI credits. See [Evals](#evals). + ## Make it yours You need Node.js 22.14 or newer. @@ -73,12 +104,11 @@ You need Node.js 22.14 or newer. 4. Add your first plugin with `npm run new-plugin -- `. See [Add a plugin](#add-a-plugin). `npm run verify` passes even with no plugins at all. - -5. Decide whether to keep the remote `github-copilot` plugin in the catalog. It - needs access to the private `github/evals` repository; see + +5. Decide whether to keep the remote `eval` plugin in the catalog. See [Remote plugins](docs/remote-plugins.md). Remove it with - `npm run remove-plugin -- github-copilot`. - + `npm run remove-plugin -- eval`. + 6. Replace what's meant for you before you ship: - Your own copyright line in [LICENSE](LICENSE), beside the template's. That one file covers the whole repository, including every plugin. See @@ -130,11 +160,6 @@ copilot plugin install json-configuration@plugin-marketplace-starter Choose only the plugins you need; each one installs on its own. Start a new session after installing. - -This repository is currently private, so installers need a GitHub account with -access to it. - - ### VS Code Run **Chat: Install Plugin from Source** and enter the repository, here @@ -234,10 +259,9 @@ marketplace consistent. [evals.yml](.github/workflows/evals.yml) runs behavioral evals. It's off for pull requests until you set the repository variable `ENABLE_EVALS` to `true`, and you can always run it by hand. Each run can wait for a reviewer's approval -through the `evals` environment. By default it uses the private -[`github/evals-action`](https://github.com/github/evals-action), which only -repositories in the GitHub organization can use. Anywhere else, including a -personal account, set the repository variable `EVALS_RUNNER` to `npm` to run +through the `evals` environment. By default it uses +[`github/evals-action`](https://github.com/github/evals-action). Set the +repository variable `EVALS_RUNNER` to `npm` to run the same evals with the pinned Vally CLI. On a personal account, usage is billed to your own Copilot plan. [Continuous integration](docs/ci.md) covers setup, personal accounts, billing, cost, and security. @@ -291,6 +315,27 @@ The `Copyright` line in `LICENSE` says who holds the copyright in the code. [license](schemas/LICENSE-APACHE-2.0.txt) and [notice](schemas/NOTICE), and the third-party notices beside each generated bundle. + +## Maintainers + +GitHub's Copilot Evals team, `@github/code-ai-evals`, maintains this template. +[.github/CODEOWNERS](.github/CODEOWNERS) lists the code owners who review +changes. + +## Support + +See [SUPPORT.md](SUPPORT.md) for how to ask questions and report bugs, and +[SECURITY.md](SECURITY.md) for how to report a vulnerability privately. + +## Acknowledgement + +This template builds on the [Agent Plugins](https://agent-plugins.org/) +standard and its schemas, and runs its evals with +[Vally](https://microsoft.github.io/vally/). Thanks to +[@digitarald](https://github.com/digitarald), who wrote its first version, and +to everyone who's contributed since. + + ## References - [Customization types](docs/customization-types.md), [Hooks](docs/hooks.md), diff --git a/SECURITY.md b/SECURITY.md index c8da47f..68e5f93 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,6 +4,34 @@ Don't report security problems in public issues, discussions, or pull requests. + +Thanks for helping make GitHub safe for everyone. + +GitHub takes the security of our software products and services seriously, including all of the open source code repositories managed through our GitHub organizations, such as [GitHub](https://github.com/GitHub). + +Even though [open source repositories are outside of the scope of our bug bounty program](https://bounty.github.com/index.html#scope) and therefore not eligible for bounty rewards, we will ensure that your finding gets passed along to the appropriate maintainers for remediation. + +If you believe you have found a security vulnerability in any GitHub-owned repository, please report it to us through coordinated disclosure. + +**Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.** + +Instead, please send an email to opensource-security[@]github.com. + +Please include as much of the information listed below as you can to help us better understand and resolve the issue: + + * The type of issue (e.g., buffer overflow, SQL injection, or cross-site scripting) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +See [GitHub's Safe Harbor Policy](https://docs.github.com/en/site-policy/security-policies/github-bug-bounty-program-legal-safe-harbor#1-safe-harbor-terms) + + Use this repository's **Security** tab to [report a vulnerability privately](https://docs.github.com/en/code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository), diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..73e153a --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,15 @@ +# Support + +## How to file issues and get help + +This project uses GitHub issues to track bugs and feature requests. Please search the existing issues before filing new issues to avoid duplicates. For new issues, file your bug or feature request as a new issue. + +For help or questions about using this project, please open an issue. + + +The plugin marketplace starter is under active development and maintained by GitHub staff and the community. We will do our best to respond to support, feature requests, and community questions in a timely manner. + + +## GitHub Support Policy + +Support for this project is limited to the resources listed above. diff --git a/docs/ci.md b/docs/ci.md index 48588ad..4b6a814 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -40,8 +40,7 @@ organization: - `validate.yml` works as it is, and it's all you need to keep the marketplace consistent. It never calls a model. - To run behavioral evals in CI, set the repository variable `EVALS_RUNNER` to - `npm`. The default runner uses a private Action that only repositories in the - GitHub organization can use, so without this the eval job fails. + `npm`. - Copilot usage from the job's token is [billed to your own Copilot plan](https://docs.github.com/en/copilot/concepts/agent-capabilities/agent-powered-workflows-and-operations/copilot-cli-in-github-actions). Each plan includes a monthly allowance of AI credits; see @@ -106,13 +105,13 @@ requests as drafts. |---|---|---| | Set `EVALS_RUNNER` to | nothing, or `action` | `npm` | | Runs | [`github/evals-action`](https://github.com/github/evals-action) v0.1.0 | `node scripts/evals.mjs eval --suite --require-pass` | -| Who can use it | Repositories in the GitHub organization. The Action is private and shared only there, so other organizations can't use it. | Any repository | +| Who can use it | Any repository | Any repository | | Report | Structured report and one sticky pull request comment | The job summary | | Job token permissions | `contents: read`, `copilot-requests: write`, `pull-requests: write` | `contents: read`, `copilot-requests: write` | The two runners are separate jobs, because a job downloads every action it names before it starts, even ones in skipped steps. With `EVALS_RUNNER=npm`, the -private Action is never downloaded. +Action is never downloaded. Both fail the job on a failed verdict, an incomplete run, or an infrastructure error, unlike the report-only default of `npm run eval:report` locally. Both use diff --git a/docs/remote-plugins.md b/docs/remote-plugins.md index f89180d..417279f 100644 --- a/docs/remote-plugins.md +++ b/docs/remote-plugins.md @@ -11,23 +11,23 @@ run upstream code, and the local evals don't cover remote plugins: `evals/` is for the plugins under `plugins/`, and the coverage check rejects asset IDs that aren't local. Evaluate an upstream plugin in its own repository. - + ## The GitHub Copilot Evals plugin -[marketplace.json](../marketplace.json) includes a `github-copilot` entry that +[marketplace.json](../marketplace.json) includes an `eval` entry that points at the published GitHub Copilot Evals plugin in [github/evals](https://github.com/github/evals). It mirrors the stable entry in -upstream's own catalog: the `copilot-evals-v0.16.0` release, pinned to the full +upstream's own catalog: the `copilot-evals-v0.20.3` release, pinned to the full SHA of its built commit. After adding this marketplace, install it explicitly with -`copilot plugin install github-copilot@MARKETPLACE`, where `MARKETPLACE` is the +`copilot plugin install eval@MARKETPLACE`, where `MARKETPLACE` is the `name` in `marketplace.json`. Or skip this catalog and install it from upstream, which follows upstream's reviewed stable releases: ```sh copilot plugin marketplace add github/evals -copilot plugin install github-copilot@evals +copilot plugin install eval@github-copilot-evals-dist ``` Pick one route. Don't install the same plugin from both marketplaces. @@ -35,16 +35,12 @@ Pick one route. Don't install the same plugin from both marketplaces. The pinned release is the built plugin. It provides the native `/eval` command (`/eval setup`, `create`, `review`, `improve`, `run`, and `compare`) in the CLI and Copilot App, a canvas, and six model-invocable skills. Running an eval -needs a compatible runner and its own approval, and it spends AI credits. See -the [upstream install guide](https://github.com/github/evals/blob/main/website/src/content/guides/install/plugin.md) -for VS Code and Copilot App setup. This repository's eval tooling works without -it. - -The `github/evals` repository is currently private, so each installer needs a -GitHub account with access. This catalog entry doesn't grant access or copy -upstream code. If your team doesn't have access, remove the entry with -`npm run remove-plugin -- github-copilot`. - +needs a compatible runner and its own approval, and it spends AI credits. +This repository's eval tooling works without it. + +This catalog entry doesn't copy upstream code. If you don't want it, remove the +entry with `npm run remove-plugin -- eval`. + ## Add a remote plugin diff --git a/docs/security.md b/docs/security.md index 8484d20..6878e44 100644 --- a/docs/security.md +++ b/docs/security.md @@ -32,7 +32,7 @@ plugins. | Local eval results in `vally-results/`: results, a report, JUnit, and session evidence with transcripts | Your disk | Gitignored. Delete them when you're done. | | npm runner in CI: Vally's log and the report in the job summary | The workflow run | Anyone who can read the repository's Actions runs. Logs follow the repository's [retention setting](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization) (90 days by default). | | npm runner results artifact, which includes transcripts | A workflow artifact | Only when `EVALS_UPLOAD_RESULTS` is `true`. Kept for 7 days. | -| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. The Action is private, and this template doesn't document its internals. | +| Evals Action report and pull request comment | The workflow run and the pull request | Anyone who can read them. The report and comment hold case names, verdicts, and grader evidence. Raw transcript upload is off. This template doesn't document the Action's internals. | | Vally telemetry | Vally's telemetry service, unless turned off | `VALLY_TELEMETRY_OPTOUT=1` is set by `validate.yml`, the local launcher, and both eval jobs. The Action job also forwards it to Vally's run phase; whether the Action's lint and report phases honor it isn't verified. | | Copilot CLI telemetry | GitHub | Set by Copilot's own settings and policies, not by this template. | diff --git a/marketplace.json b/marketplace.json index 754e01e..60fe4a5 100644 --- a/marketplace.json +++ b/marketplace.json @@ -39,15 +39,15 @@ "description": "Validate JSON and retrieve configuration values with a local MCP server." }, { - "name": "github-copilot", + "name": "eval", "source": { "source": "github", "repo": "github/evals", - "ref": "copilot-evals-v0.16.0", - "sha": "dd4c5015b5e8fdfcbd17b261d996c85aa92fde5b", + "ref": "copilot-evals-v0.20.3", + "sha": "acd4b5029b0791fbe1c4b58e28fa94c580841d69", "path": "packages/plugin" }, - "version": "0.16.0", + "version": "0.20.3", "description": "Configure native projects with /eval setup, create and save evals with /eval create, save distinct candidates with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saving and running follow separate host approval policies.", "homepage": "https://github.com/github/evals#installed-skills" } diff --git a/test/adoption.test.mjs b/test/adoption.test.mjs index 9b67242..7baf7b2 100644 --- a/test/adoption.test.mjs +++ b/test/adoption.test.mjs @@ -187,7 +187,7 @@ test("an empty or remote-only marketplace passes verify, and new plugins scaffol const setup = (await readFile(join(directory, "README.md"), "utf8")).match(/## Make it yours\n[\s\S]*?\n## /)?.[0]; if (setup) { if (present.length) assert.doesNotMatch(setup, /remove-example/); - assert.doesNotMatch(setup, /github-copilot/); + assert.doesNotMatch(setup, /plugin:eval\b|remove-plugin -- eval\b/); const numbers = [...setup.matchAll(/^(\d+)\. /gm)].map((match) => Number(match[1])); assert.deepEqual(numbers, numbers.map((_, index) => index + 1)); } diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs index 947decb..7f0209c 100644 --- a/test/catalog.test.mjs +++ b/test/catalog.test.mjs @@ -12,7 +12,7 @@ test("the catalog separates local plugins from pinned, opt-in remote references" const evals = remotePlugins.find((entry) => entry.source.repo === "github/evals"); if (evals) { // The catalog name must match upstream's plugin.json name, not its marketplace name. - assert.equal(evals.name, "github-copilot"); + assert.equal(evals.name, "eval"); assert.equal(evals.source.path, "packages/plugin"); assert.deepEqual(remoteWarnings(evals), []); assert.equal(evals.version, evals.source.ref.replace(/^copilot-evals-v/, "")); diff --git a/test/eval-setup.test.mjs b/test/eval-setup.test.mjs index b437c4c..66f9b31 100644 --- a/test/eval-setup.test.mjs +++ b/test/eval-setup.test.mjs @@ -300,7 +300,7 @@ test("deterministic CI stays model-free, and opt-in behavioral CI uses the Evals } const npmJob = evals.jobs["evals-npm"]; if (npmJob) { - assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the private Action."); + assert.ok(!npmJob.steps.some((step) => step.uses?.startsWith("github/evals-action@")), "The npm runner must not need the Action."); const runStep = npmJob.steps.find((step) => /scripts\/evals\.mjs eval\b/.test(step.run ?? "")); assert.ok(runStep.run.includes("--require-pass")); const report = npmJob.steps.find((step) => /eval-report\.mjs/.test(step.run ?? ""));