From 7330e07da4e33b8692be408d24e2c004411a4d00 Mon Sep 17 00:00:00 2001 From: Cassondra Roberts <1840295+castastrophe@users.noreply.github.com> Date: Fri, 6 Nov 2026 20:35:03 +0000 Subject: [PATCH] docs: require code coverage & PR coverage reports Every repository with tests reports coverage with c8, Vitest's v8 provider, or the host runner's own, & posts one summary comment per pull request from the Node 26 job, as envoy does. The matrix gets a Coverage check in place of the undecided manual row. --- AGENTS.md | 1 + docs/repository-standards.md | 34 +++++++++++++++++-- docs/testing.md | 23 +++++++++++-- skills/standards-matrix/SKILL.md | 2 +- skills/standards-matrix/data/manual.json | 22 ------------ skills/standards-matrix/references/checks.md | 1 + skills/standards-matrix/scripts/lib/checks.js | 24 +++++++++++++ skills/standards-matrix/test/checks.test.js | 31 +++++++++++++++++ skills/standards-matrix/test/scan.test.js | 4 +-- 9 files changed, 113 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 83df08d..bbc2c0b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,6 +108,7 @@ Full rules in [`docs/testing.md`](docs/testing.md). - `describe` names the unit; `it` states the behavior in the present tense, lowercase, without "should". - New behavior & bug fixes ship with a test in the same PR. +- Coverage is reported on every pull request (c8, or Vitest's v8 provider, with a 50% line floor). ## Repository configuration diff --git a/docs/repository-standards.md b/docs/repository-standards.md index 808f996..31e986c 100644 --- a/docs/repository-standards.md +++ b/docs/repository-standards.md @@ -19,7 +19,7 @@ note allows. | `.env.example` | `.env.example` | Lists every variable the repo reads; required only where it reads some | | `.github/renovate.json` | `renovate.json` | One-month release age, monthly lockfile refresh | | `.github/release.yml` | `release.yml` | Changelog categories keyed on `type:` labels | -| `.gitignore` | `.gitignore` | Add `_site` for repos that build an Eleventy site | +| `.gitignore` | `.gitignore` | Add `_site` for repos that build an Eleventy site, & `coverage` for every repo with tests | Nothing syncs these files. The Weaver workflow ([`sync.yaml`](../.github/workflows/sync.yaml)) updates only the README sections from `templates/`, so a config change here needs a PR in @@ -179,7 +179,7 @@ on the same files where it is not adopted yet. | File | Rule | | ----------------------------- | -------------------------------------------------------------------------------------------- | -| `.github/workflows/ci.yml` | On `pull_request` & pushes to `main`. `permissions: contents: read`. Steps: checkout with `persist-credentials: false`, `corepack enable`, `setup-node` with `node-version: ${{ matrix.node }}` & `cache: yarn`, `yarn install --immutable`, `yarn lint`, `yarn build`, `yarn typecheck`, `yarn test`. One job named `CI (node${{ matrix.node }})` over `node: [26, 24]`, so the checks are `CI (node26)` & `CI (node24)`; both are required | +| `.github/workflows/ci.yml` | On `pull_request` & pushes to `main`. `permissions: contents: read`. Steps: checkout with `persist-credentials: false`, `corepack enable`, `setup-node` with `node-version: ${{ matrix.node }}` & `cache: yarn`, `yarn install --immutable`, `yarn lint`, `yarn build`, `yarn typecheck`, `yarn test` (`yarn coverage` on the Node 26 job, with the coverage steps below). `permissions` add `pull-requests: write` for the coverage comment. One job named `CI (node${{ matrix.node }})` over `node: [26, 24]`, so the checks are `CI (node26)` & `CI (node24)`; both are required | | `.github/workflows/pages.yml` | Calls the org's reusable `pages.yml@main` with `output-dir`; grants `contents: write` & `pull-requests: write` | | `.github/renovate.json` | The baseline above, with `reviewers` naming the maintainer | @@ -206,6 +206,36 @@ jobs: cache: yarn ``` +Coverage steps replace the single test step. They run on pull requests from the Node 26 +job only; see [Coverage](testing.md#coverage) for the tools & the floor. + +```yaml + - name: Test + if: matrix.node != 26 + run: yarn test + + - name: Test with coverage + if: matrix.node == 26 + run: yarn coverage + + - name: Coverage summary + uses: irongut/CodeCoverageSummary@v1.3.0 + if: matrix.node == 26 && github.event_name == 'pull_request' + with: + filename: coverage/cobertura-coverage.xml + badge: true + fail_below_min: true + format: markdown + output: both + + - name: Coverage comment + uses: marocchino/sticky-pull-request-comment@v3.0.4 + if: matrix.node == 26 && github.event_name == 'pull_request' + with: + recreate: true + path: code-coverage-results.md +``` + ## Documentation | File | Rule | diff --git a/docs/testing.md b/docs/testing.md index 88152b4..051f10b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -12,6 +12,7 @@ files & settings live in [repository-standards.md](repository-standards.md). | Ecosystem-defined | The host's recommended technique | The tool a plugin extends documents a test approach: follow it over the two defaults | | Accessibility & rendered pages | [Cypress](https://www.cypress.io) & [cypress-axe](https://github.com/component-driven/cypress-axe) | Any site or UI; WCAG 2.2 AA in light, dark, & forced-colors. See [`oss-docs-site` testing](../skills/oss-docs-site/references/testing.md) | | Types | `tsc` through `yarn build` & `yarn typecheck` | Every TypeScript package; `typecheck` covers `src` & `test` | +| Coverage | [c8](https://github.com/bcoe/c8) for AVA & `node:test`; `@vitest/coverage-v8` for Vitest; the host runner's own for ecosystem-defined tests | Every repository with a `test` script; CI reports it on each pull request | | Style & license headers | `opx lint` | Every repository; CI runs it first | Python projects use pytest. @@ -103,6 +104,23 @@ root: Yarn 4 resolves a workspace's binaries from its own dependencies, & only t `test` script puts the root's binaries on `PATH`. Declare `vitest` in each package that runs it if that form is wanted. +## Coverage + +Every repository with a `test` script reports code coverage, & CI shows it on each pull +request. [envoy](https://github.com/allonsy-studio/envoy) is the reference. + +| Part | Rule | +| --------- | ------------------------------------------------------------------------------------------------------ | +| Script | A root `coverage` script runs every workspace's tests with coverage on | +| Tool | AVA & `node:test`: `"coverage": "c8 yarn test"`. Vitest: `"coverage": "vitest run --coverage"` with `@vitest/coverage-v8`. Jest, where the host ecosystem uses it: `"coverage": "jest --coverage"` | +| Reporters | `text` for the log & `cobertura`, which writes `coverage/cobertura-coverage.xml` | +| Ignore | `coverage/` in `.gitignore` | +| Report | One sticky comment per pull request: [CodeCoverageSummary](https://github.com/irongut/CodeCoverageSummary) renders the XML, [sticky-pull-request-comment](https://github.com/marocchino/sticky-pull-request-comment) posts it | +| Floor | `fail_below_min: true` fails the check under a 50% line rate. A repository may raise the floor, never lower it | + +Only the Node 26 job runs `yarn coverage` & posts the comment; the Node 24 job runs +`yarn test`. The [CI snippet](repository-standards.md#ci--automation) has the steps. + ## What gets a test 1. Every exported function or CLI flag: a test in the same PR. @@ -122,11 +140,12 @@ The PR template's checklist asks for the same. 5. TypeScript packages: add `tsconfig.test.json` (extends the base, `noEmit`, includes every `src` & `test`) & a `typecheck` script. 6. Add `typecheck` & `test` to the CI job after `build`. -7. Add the Testing section to the repository's `AGENTS.md`, linking this file. +7. Add a `coverage` script, the coverage tool, & the CI steps from the [Coverage](#coverage) section. +8. Add the Testing section to the repository's `AGENTS.md`, linking this file. ## Known gaps | Where | Gap | | ----------------------------- | ------------------------------------------------------------------------------------------ | | This repo's `skills/*/test/` | The suites use `node:test`, not AVA | -| Coverage | No tool or threshold is set anywhere; none is required until one is chosen | +| Coverage | Reporting is uneven; the matrix lists each repository that lacks a part of it | diff --git a/skills/standards-matrix/SKILL.md b/skills/standards-matrix/SKILL.md index 8551318..3eff8b9 100644 --- a/skills/standards-matrix/SKILL.md +++ b/skills/standards-matrix/SKILL.md @@ -48,7 +48,7 @@ Rules each check enforces: `references/checks.md`. The standards themselves: [`d ``` `--previous` takes the published page (read it first: the Artifact `read` action names the saved file) or an earlier `matrix.json`. The scan prints counts per repository & every row that changed status. A row is `pr` only when an open pull request head passes a check that the default branch fails. -5. **Manual rows & exemptions.** Every row with `manual: true` shows a `verified` date. Ask about each row older than 14 days, & any the maintainer says changed, with one question per row: coverage (undecided, or tool & threshold), required checks (set, or not), attribution footers (cleaned, or how many remain). Edit a copy of `data/manual.json` in the scratchpad & pass it with `--manual`; change the default file only through a pull request. Never carry a manual status forward unasked. A scope decision (a tool that does not apply to a repository) goes in `data/exemptions.json` with the reason & the date, only on the maintainer's say-so, & shows as Not applicable. The scan warns about an entry that matches nothing: fix the name. +5. **Manual rows & exemptions.** Every row with `manual: true` shows a `verified` date. Ask about each row older than 14 days, & any the maintainer says changed, with one question per row: required checks (set, or not), attribution footers (cleaned, or how many remain). Edit a copy of `data/manual.json` in the scratchpad & pass it with `--manual`; change the default file only through a pull request. Never carry a manual status forward unasked. A scope decision (a tool that does not apply to a repository) goes in `data/exemptions.json` with the reason & the date, only on the maintainer's say-so, & shows as Not applicable. The scan warns about an entry that matches nothing: fix the name. 6. **Build & publish.** `node scripts/build.js --data matrix.json --out matrix.html`. Read the published page first (a publish to an unread artifact is refused), then publish `matrix.html` with that `url`. An update keeps the link & icon. 7. **Report.** Counts per repository, the rows that changed since the last scan, the rows that need the maintainer, & what the scan could not see (settings, fork pull requests, repositories not checked out). Say that the page is private & who can open it. diff --git a/skills/standards-matrix/data/manual.json b/skills/standards-matrix/data/manual.json index b149f61..7625a79 100644 --- a/skills/standards-matrix/data/manual.json +++ b/skills/standards-matrix/data/manual.json @@ -1,26 +1,4 @@ [ - { - "area": "Testing", - "tool": "Coverage", - "std": "No tool or threshold is chosen; none is required until one is.", - "src": "", - "repos": { - ".github": { - "status": "you", - "state": "Undecided.", - "verified": "2026-11-06" - }, - "webc-docs": { - "status": "you", - "state": "Undecided.", - "verified": "2026-11-06" - } - }, - "default": { - "status": "you", - "state": "Undecided." - } - }, { "area": "Settings", "tool": "Required checks", diff --git a/skills/standards-matrix/references/checks.md b/skills/standards-matrix/references/checks.md index 1a2a3b5..8a32cb5 100644 --- a/skills/standards-matrix/references/checks.md +++ b/skills/standards-matrix/references/checks.md @@ -14,6 +14,7 @@ Each row of the matrix comes from one check in `scripts/lib/checks.js`. A check | `internal-ranges` | Internal dependency ranges | A package depends on another local package with a `workspace:` range | No package depends on another | | `test-runner` | Test runner | A package runs `node:test`, or a TypeScript package does not run Vitest | No package defines a test script | | `test-layout` | Test layout | A folder is named `tests/` outside samples | No test folders | +| `coverage` | Coverage | A package has a test script & the repository lacks a root `coverage` script, `c8`, `@vitest/coverage-v8`, or `jest`, a CI step that runs coverage, the `CodeCoverageSummary` & `sticky-pull-request-comment` steps, or a `coverage` line in `.gitignore` | No package defines a test script | | `license-header` | License header | A file under `src/`, `test/`, or `bin/` lacks an SPDX line in its first 400 characters, or nothing enforces it | No such files | | `typecheck` | Typecheck | TypeScript exists without `tsconfig.test.json`, a `typecheck` script, or a CI step | No TypeScript | | `a11y` | Accessibility audit | A `site/` or `docs/eleventy.config.js` exists without `cypress-axe` or a Cypress audit | No site | diff --git a/skills/standards-matrix/scripts/lib/checks.js b/skills/standards-matrix/scripts/lib/checks.js index 09c0753..3861409 100644 --- a/skills/standards-matrix/scripts/lib/checks.js +++ b/skills/standards-matrix/scripts/lib/checks.js @@ -28,6 +28,9 @@ const ORG_ROOT_NAME = "@allons-y/core"; /** Variables every environment provides, so `.env.example` need not list them. */ const AMBIENT_ENV = new Set(["CI", "NODE_ENV", "HOME", "PATH", "TMPDIR", "TEMP", "TMP", "USER", "SHELL", "NO_COLOR", "FORCE_COLOR", "NODE_OPTIONS", "NODE_PATH", "PWD", "DEBUG"]); +/** Dependencies that provide coverage: c8, Vitest's v8 provider, & Jest's built-in reporter. */ +const COVERAGE_TOOLS = ["c8", "@vitest/coverage-v8", "jest"]; + const tsSourceFiles = (repo) => repo.files().filter((f) => TS_FILE.test(f) && !f.endsWith(".d.ts") && !isFixture(f)); const hasTypeScript = (repo) => tsSourceFiles(repo).length > 0; const tsPackages = (repo) => repo.packages().filter((p) => repo.has(p.dir === "." ? "tsconfig.json" : `${p.dir}/tsconfig.json`)); @@ -211,6 +214,27 @@ export const CHECKS = [ return any ? ok("test/ folders beside the source.") : na("No test folders."); }, }, + { + id: "coverage", + area: "Testing", + tool: "Coverage", + std: "A coverage script, c8, @vitest/coverage-v8, or Jest, a CI step, a PR summary & comment, & coverage/ ignored.", + src: "package.json · ci.yml · .gitignore", + run(repo) { + const packages = repo.packages(); + if (!packages.some((p) => scriptsOf(p).test)) return na("No package defines a test script."); + const ci = allWorkflows(repo); + const problems = []; + if (!scriptsOf({ json: rootPkg(repo) }).coverage) problems.push("no root coverage script"); + if (!packages.some((p) => COVERAGE_TOOLS.some((tool) => depsOf(p)[tool]))) problems.push("no c8, @vitest/coverage-v8, or jest dependency"); + if (!runsStep(ci, "coverage")) problems.push("no CI step runs coverage"); + if (!/CodeCoverageSummary/.test(ci)) problems.push("no coverage summary step"); + if (!/sticky-pull-request-comment/.test(ci)) problems.push("no coverage comment step"); + if (!/^\/?coverage\/?\s*$/m.test(repo.read(".gitignore") ?? "")) problems.push("coverage/ is not ignored"); + if (problems.length) return gap(sentence(problems)); + return ok("Coverage script, tool, CI report, & ignore are in place."); + }, + }, { id: "license-header", area: "Testing", diff --git a/skills/standards-matrix/test/checks.test.js b/skills/standards-matrix/test/checks.test.js index 1aa8dd8..6fa2ba7 100644 --- a/skills/standards-matrix/test/checks.test.js +++ b/skills/standards-matrix/test/checks.test.js @@ -217,3 +217,34 @@ test("ci: literal job names count", (t) => { const literal = CI_BOTH.replace("CI (node${{ matrix.node }})", "CI (node26)"); t.is(runCheck("ci", { ".github/workflows/ci.yml": literal })[0], "ok"); }); + +const COVERAGE_CI = `${CI_BOTH} - run: yarn coverage\n - uses: irongut/CodeCoverageSummary@v1.3.0\n - uses: marocchino/sticky-pull-request-comment@v3.0.4\n`; +const COVERAGE_FILES = { + "package.json": { scripts: { test: "ava", coverage: "c8 yarn test" }, devDependencies: { c8: "^12.0.0" } }, + ".gitignore": "node_modules\ncoverage\n", + ".github/workflows/ci.yml": COVERAGE_CI, +}; + +test("coverage passes with the script, tool, CI report, & ignore", (t) => { + t.is(runCheck("coverage", COVERAGE_FILES)[0], "ok"); +}); + +test("coverage accepts Vitest's v8 provider", (t) => { + const files = { ...COVERAGE_FILES, "package.json": { scripts: { test: "vitest run", coverage: "vitest run --coverage" }, devDependencies: { "@vitest/coverage-v8": "^4.0.0" } } }; + t.is(runCheck("coverage", files)[0], "ok"); +}); + +test("coverage lists what a repository with tests is missing", (t) => { + const [status, state] = runCheck("coverage", { "package.json": { scripts: { test: "ava" } }, ".github/workflows/ci.yml": CI_BOTH }); + t.is(status, "gap"); + for (const part of [/root coverage script/, /c8, @vitest\/coverage-v8, or jest/, /CI step runs coverage/, /summary step/, /comment step/, /not ignored/]) t.regex(state, part); +}); + +test("coverage accepts Jest's own reporter", (t) => { + const files = { ...COVERAGE_FILES, "package.json": { scripts: { test: "jest", coverage: "jest --coverage" }, devDependencies: { jest: "^30.0.0" } } }; + t.is(runCheck("coverage", files)[0], "ok"); +}); + +test("coverage is not applicable without a test script", (t) => { + t.is(runCheck("coverage", { "package.json": { scripts: {} } })[0], "na"); +}); diff --git a/skills/standards-matrix/test/scan.test.js b/skills/standards-matrix/test/scan.test.js index 02576d4..3483f19 100644 --- a/skills/standards-matrix/test/scan.test.js +++ b/skills/standards-matrix/test/scan.test.js @@ -48,9 +48,9 @@ test("manual rows appear for the repositories they name & carry their flag", (t) }); test("a manual default applies to every repository the item does not name", (t) => { - const manual = [{ area: "Testing", tool: "Coverage", std: "x", repos: { alpha: { state: "Chosen." } }, default: { state: "Undecided." } }]; + const manual = [{ area: "Testing", tool: "Branch protection", std: "x", repos: { alpha: { state: "Chosen." } }, default: { state: "Undecided." } }]; const scan = runScan({ repos: [{ name: "alpha", dir: tempRepo(BEFORE) }, { name: "beta", dir: tempRepo(BEFORE) }], manual, asOf: "2026-11-06" }); - const state = (repo) => scan.rows.find((r) => r.repo === repo && r.tool === "Coverage").state; + const state = (repo) => scan.rows.find((r) => r.repo === repo && r.tool === "Branch protection").state; t.is(state("alpha"), "Chosen."); t.is(state("beta"), "Undecided."); });