From 9ab1fc6255e01d0f367190f2a20efc8ba5116759 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 18:59:57 -0700 Subject: [PATCH 1/8] fix(assurance): refresh the catalog checksums #491 invalidated #491 rewrote smoke goldens so they stop pinning the tool version, and landed on main between the last sync of #398 and its merge. The two touched different files, so they merged cleanly, but the catalog's claims pin the checksum of the golden that backs each of them, and fifteen of those now named a file that had changed underneath them. The first `Release prerequisites` run on main caught it: `catalog-valid` failed on `finding-baseline-workflow.golden.json`. That is the gate working as designed, and it would have refused the next tag. Refreshed with `catalog-validate --refresh`; no claim changed, only the recorded checksums. Co-Authored-By: Claude Fable 5.1 --- docs/assurance/catalog.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index 2188fa51..091e1d84 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -760,7 +760,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", - "sha256": "0393003fb3d39f8c923325062bbb257606c96a9f60e278cad64c106527ebaef7" + "sha256": "84fb94c8cec1b0399259be84c5d9b252d8f4df3c15b0ac813eca2c765c363ae1" } ], "proves": [ @@ -803,7 +803,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", - "sha256": "b17ad417464c74d9c8705177d3b067c8d20ab63d382ee4476ad1da9e2748d8a5" + "sha256": "6fc075beb7e3eca64c9ca8e79c07ff0160e3ff8a3f4284b8df813ef927ce5b9e" } ], "proves": [ @@ -890,7 +890,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-bun.golden.json", - "sha256": "beae880e2ec43d1aa5c0da09bc86903ffc370638ab5b416bf521dfb60d9a8693" + "sha256": "e4df856333b558128b07165cc16d44a12d9981bbd42054ad2912cfb699a563a5" } ], "proves": [ @@ -938,7 +938,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-go.golden.json", - "sha256": "1352983b2aafc221b9058a82c3edd1e1f8d983b5908ecbcd01a5e0cc11233644" + "sha256": "267270c11e350f668c4e018c7d36e675c7412b11d37aec5725380fabb391e2a8" } ], "proves": [ @@ -987,7 +987,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-maven.golden.json", - "sha256": "96feaa39866e32319b9e0009541afb59f1b856515636ffdc72fb1a5a1b4107f4" + "sha256": "7093935763fe2735f42a475ad22c6af6284beb352f3bed02f87521a4f48f383d" } ], "proves": [ @@ -1035,7 +1035,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-npm.golden.json", - "sha256": "96f65ace60a7b75087bdad36bda45959aa769ac4dd2c57f13cdc8983b11ed4be" + "sha256": "bb19b6686664f1ce4ba3424168a417b46c89e5680e84f996c879b40281451527" } ], "proves": [ @@ -1083,7 +1083,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", - "sha256": "972452883100713d6cc20d1ba5739b548940288571a4bca240b96fa14bcc586e" + "sha256": "eb67f160516425a5828bc546589dc468d50f37525e47e4dc793a0d166bbc4286" } ], "proves": [ @@ -1130,7 +1130,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", - "sha256": "9a5edd634c17c6b04c51135b1c954ae509043f484dadf49c46427322f7bcc9a9" + "sha256": "38a550af6decc30523da724abd78acddb2e7b659df4651eba8313bd9000a9fd9" } ], "proves": [ @@ -1178,7 +1178,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-yarn.golden.json", - "sha256": "8aacecd44516a5cda5a4d84bdfbedb982095924580119d8d7cc8b5f2de6dd57e" + "sha256": "de649973a762167d0d474532773226c284330ee2bcaab4f6fa6dbd26867e34b2" } ], "proves": [ @@ -1306,7 +1306,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", - "sha256": "0e66a3ef13f9555d326f385fac977a910ab2dbe21da5d05affef4ab03e0b7cb6" + "sha256": "9d70ea49a7c99e9dc2c2bee0ad4c1dc0368bbe02167386d62eb8bb793f39b5ae" } ], "proves": [ @@ -1355,7 +1355,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", - "sha256": "9e8f69f58dfab4ff9e0e41dec928c57468d7731ad1a981b904f0bf97d2cac5a6" + "sha256": "60060bf892235feef6df1e2d18e69cd1a00b6a277e8d3537a4dad4fed0a6c022" } ], "proves": [ @@ -1403,7 +1403,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", - "sha256": "2ea78dc8dc40f0aebf84442300672481c4feaa24c0b5abdf012a7da4c06874fd" + "sha256": "f9774f51e9e9c3a931a900c9be62b93b5810cf023f500ebf9a96f6f1fdeb2a66" } ], "proves": [ @@ -1450,7 +1450,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", - "sha256": "181b80b3f2807d6607ccedfa8aafc99b370a78d7161c540316b8a323765be56d" + "sha256": "8a40d041d23f4bc4984bb3ba00cc3919592fddcd694b45855e0fb45d5d73077b" } ], "proves": [ @@ -1536,7 +1536,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", - "sha256": "726d17483521ff675cc8610915e16f701317f36c15f70fe8cb408f1830a3ea2e" + "sha256": "96a1a56860eb1c28bcb0da4d01809f74a7b07e6d0bcf637f2b8e36c6ecd9208e" } ], "proves": [ @@ -1582,7 +1582,7 @@ "artifacts": [ { "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", - "sha256": "5197ec06425ab1715a6646b230319d25d8eb5409bf08da7a2657e571175d8eee" + "sha256": "a0bb88cb6bbdbbe49e02ca6e091d15953be4320dd82c1dedcff32085dfbf042a" } ], "proves": [ From 1b13f7d6e392f43293f7b8985f0fa9b92c0b5401 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 19:04:25 -0700 Subject: [PATCH 2/8] fix(assurance): a claim names its files by path, with no stored checksum The catalog stored a SHA-256 for every fixture and expected-result file a claim names. Those files are regenerated routinely, so every golden update left the catalog stale, and the `catalog-valid` gate then refused the next release until someone rewrote the checksums. The previous commit on this branch did that rewrite. This one removes the reason for it. A file in this repository is already content-addressed by git: a release tag and a path identify exact bytes, and the report records the commit. The stored checksum was a second, hand-kept copy of what git knows. It could not catch anything git does not, and the only thing it could do was fall behind. It could also not be kept in step by checking earlier. `make test` already verified the checksums on every pull request. #491 changed goldens and #398 added the catalog; each passed CI on its own, they touched different files so they merged cleanly, and `main` failed the gate. A refresh step in `Update Smoke Goldens` only helps when that workflow is what changes a golden. So the catalog names files by path only, `catalog-validate --refresh` and its step in the goldens workflow are gone, and that workflow no longer needs Go. What is checked is what can actually go wrong: a claim naming a file that was renamed or deleted fails `make assurance-catalog` and `TestRepositoryCatalogIsValid` in `make test`, and both refuse a run that checked no files at all (ADR-0044). Still pinned, because this repository's history cannot address them: a git input by revision and a container by digest. The catalog and report keep their schema versions. No report has been published, the site does not read the removed field, and the catalog has no reader outside this repository. ADR-0047 carries a dated amendment. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/assurance-prerequisites.yml | 4 +- .github/workflows/update-smoke-goldens.yml | 20 +-- AGENTS.md | 4 +- dev-docs/RELEASE_ASSURANCE.md | 25 ++-- ...ance-is-a-catalog-and-a-result-contract.md | 12 ++ docs/ASSURANCE.md | 6 +- docs/assurance/catalog.json | 98 +++++-------- internal/assurance/catalog.go | 134 ++++++------------ internal/assurance/catalog_test.go | 97 +++++++------ internal/assurance/cmd/report.go | 34 ++--- internal/assurance/testdata/catalog.json | 98 ++++++++++--- .../testdata/golden/all-pass.report.json | 3 +- .../testdata/golden/mixed-failure.report.json | 3 +- 13 files changed, 249 insertions(+), 289 deletions(-) diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml index af64c556..7a2114c5 100644 --- a/.github/workflows/assurance-prerequisites.yml +++ b/.github/workflows/assurance-prerequisites.yml @@ -98,10 +98,10 @@ jobs: run: | set -euo pipefail exit_code=0 - summary="The assurance catalog is valid and every fixture and expected-result file it names still matches its recorded checksum." + summary="The assurance catalog is valid and every fixture and expected-result file it names exists." if [ "${CATALOG_OUTCOME}" != "success" ]; then exit_code=1 - summary="The assurance catalog is invalid or one of the files it names has changed. Open the validation step for the exact entry." + summary="The assurance catalog is invalid or names a file that is not there. Open the validation step for the exact entry." fi checks="$(grep -c '"stage":' docs/assurance/catalog.json || true)" # --stage and --level are explicit: when the catalog is the thing diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index e39764bd..f536b8eb 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -319,14 +319,6 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: go.mod - # No cache writes: this job runs with write permissions, and a - # poisoned cache would reach the jobs that trust it. - cache: false - - name: Download regenerated goldens uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -357,16 +349,6 @@ jobs: --jq '[.jobs[] | select(.name | startswith("Regenerate (")) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out") | .name] | join(", ")')" echo "failed=${failed}" >> "$GITHUB_OUTPUT" - - name: Refresh the catalog checksums - # Public evidence claims record the checksum of the golden file that - # backs them. Regenerating a golden without refreshing the catalog - # would fail the catalog-valid gate and block the next release. - if: steps.changes.outputs.changed == 'true' - shell: bash - run: | - set -euo pipefail - go run ./internal/assurance/cmd catalog-validate --refresh - - name: Create pull request with updated goldens if: steps.changes.outputs.changed == 'true' env: @@ -387,7 +369,7 @@ jobs: git checkout -b "${branch_name}" - git add test/smoke/testdata/golden docs/assurance/catalog.json + git add test/smoke/testdata/golden git commit -m "test: update smoke golden files" git push --set-upstream origin "${branch_name}" diff --git a/AGENTS.md b/AGENTS.md index 3800d1f8..56f8d30a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -354,7 +354,7 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - A golden must never carry a host architecture. `make smoke ARGS="-update"` on an arm64 laptop writes a golden CI cannot match, so `normalizeJSON` erases the architecture and `TestGoldensCarryNoHostArchitecture` (untagged, runs in `make test`) fails if one reaches a committed golden. A new architecture-bearing shape means teaching the normalizer, not excepting the guard. - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. - A new slice also needs an entry in the `smoke` check's `expected_instances` in `docs/assurance/catalog.json` (with its `ecosystems`, which is what puts an ecosystem on the report's coverage list). `TestCatalogSmokeInstancesMatchWorkflowMatrix` fails when the two drift. -- Regenerating goldens invalidates the checksums the catalog's claims pin. `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the catalog with them; do the same when refreshing by hand. +- Regenerating goldens needs no catalog change. A claim names its expected-result file by path only; git already records which bytes a release had, so there is no stored checksum to fall behind. Renaming or deleting a file a claim names does fail `TestRepositoryCatalogIsValid` in `make test`. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. ## Release assurance @@ -370,7 +370,7 @@ Rules: - Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. - Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. - `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. -- Public evidence claims live in the same catalog (`evidence[]`), keep their pinned Git revisions and checksummed artifacts, and name the check that backs them. `make assurance-catalog` re-hashes every file they reference. +- Public evidence claims live in the same catalog (`evidence[]`), pin what this repository's history cannot address (a Git input by revision, a container by digest), name their fixture and expected-result files by path, and name the check that backs them. `make assurance-catalog` confirms every file they name exists. Do not add a checksum of an in-repository file to the catalog: it is a second copy of what git knows, and it can only drift. - The per-release report (`docs/assurance/reports/.json`) and `docs/assurance/index.json` are written by the post-release assessment and are the only data source for the public page. See [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) for the contracts and how to add a check. diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index a6bc0f84..d33661fb 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -142,11 +142,18 @@ check `missing` — that is the intended behavior: gaps are loud. Evidence claims are the public "we prove X, we do not prove Y" statements that used to live in `test/evidence/cases.json`. They keep the same rigor: pinned -Git revisions, checksummed fixtures and expected-result files, explicit -reproduction commands, and mandatory limitations. `make assurance-catalog` -verifies every hash a claim names, so a golden file cannot drift away from the -claim it supports, and `--refresh` rewrites them when the goldens are -deliberately regenerated. +Git revisions and container digests, explicit reproduction commands, and +mandatory limitations. + +Fixture and expected-result files are named by path, with no checksum. A file +in this repository is already content-addressed by git — a release tag and a +path identify exact bytes, and the report records the commit — so a checksum +stored in the catalog was a second, hand-kept copy of something git knows. It +could only fall behind, and it did on every golden update: two pull requests +that each passed CI left `main` with a catalog that failed its own gate. What +is checked now is what can actually go wrong: `make assurance-catalog`, and +`TestRepositoryCatalogIsValid` in `make test`, fail when a claim names a file +that was renamed or deleted. A claim must carry a pinned input **and** a committed artifact. That is the line between the two layers: if a statement would only restate what its check already @@ -214,11 +221,9 @@ one is always deliberate. `assurance-prerequisites.yml` would miss every stage that Auto Version triggered. Keep that job name stable, or update the lookup in `release.yml` and `assurance-assessment.yml` with it. -- Regenerating smoke goldens changes files the evidence claims are pinned to. - `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the - catalog alongside them; do the same when refreshing goldens by hand. - That workflow regenerates from the ref it was dispatched on, so the refresh - runs the same commit's tooling against that commit's goldens. +- Regenerating smoke goldens needs no catalog change; claims name their + expected-result files by path only. Do not reintroduce a stored checksum of + an in-repository file. ## Related documents diff --git a/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md b/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md index f1b80627..63a96c12 100644 --- a/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md +++ b/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md @@ -44,6 +44,18 @@ actually download. are still checksummed, because a claim about a golden file is only as good as that file — and regenerating goldens therefore has to refresh the catalog (`catalog-validate --refresh`, wired into `Update Smoke Goldens`). + + > **Amended 2026-10-03:** the catalog no longer stores a checksum for any + > file in this repository, and `--refresh` is gone. The argument above for + > workflow files applies to goldens too: git already addresses a file by + > tag and path, so the stored checksum was a hand-kept copy that could only + > fall behind. It did, the first time the prerequisites stage ran on `main` + > — a golden update and the catalog each passed CI separately and failed + > together, which no earlier check could have caught and a refresh step only + > covers when the right workflow makes the change. A claim now names its + > files by path; what is verified is that they exist. Git inputs stay pinned + > by revision and containers by digest, because this repository's history + > cannot address those. - Reports are committed to the default branch rather than attached to the release, because the exhaustive stage runs after publication and GitHub releases are immutable once published. diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index 7c5dcbf7..7628f7d5 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -79,7 +79,7 @@ The report uses one vocabulary, and it is worth being precise about it: Some claims are asserted directly by a check. Others are asserted by comparing a pinned input — a public repository at a recorded commit, or a checked-in -fixture — against a checksummed result file, and name the check that performed +fixture — against a recorded result file, and name the check that performed that comparison. Both appear as claims on the page, in the section they belong to. A claim earns a separate entry only when it adds something the check cannot say on its own. @@ -94,8 +94,8 @@ make assurance-catalog go run ./internal/assurance/cmd catalog-validate --evidence graph-npm ``` -The first command validates the whole catalog, including the checksums of every -expected-result file it names. The second prints one claim with its reproduction +The first command validates the whole catalog and confirms every expected-result +file it names exists. The second prints one claim with its reproduction command. ## What a verified claim does not mean diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index 091e1d84..455d7857 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -79,7 +79,7 @@ "area": "framework", "stage": "prerequisites", "level": "gate", - "description": "Validates this catalog and confirms every fixture and golden file it points at still has the recorded checksum.", + "description": "Validates this catalog and confirms every fixture and expected-result file it points at exists.", "source": { "workflow": "assurance-prerequisites.yml", "job": "catalog" @@ -91,7 +91,7 @@ ] ], "proves": [ - "Every claim in this report points at a real check and at repository files that have not changed since the claim was written." + "Every claim in this report points at a real check and at files that are present in this release's source." ], "limitations": [ "A valid catalog says the paperwork is consistent; the checks themselves decide whether the software works." @@ -180,7 +180,7 @@ [ "sh", "-c", - "BOMLY_VERSION=\u003ctag\u003e BOMLY_INSTALL_DIR=$PWD/bin sh scripts/install.sh" + "BOMLY_VERSION= BOMLY_INSTALL_DIR=$PWD/bin sh scripts/install.sh" ] ], "proves": [ @@ -230,7 +230,7 @@ "gh", "release", "download", - "\u003ctag\u003e", + "", "--repo", "bomly-dev/bomly-cli" ] @@ -264,7 +264,7 @@ "--dir", "assets", "--version", - "\u003cversion\u003e" + "" ] ], "proves": [ @@ -308,7 +308,7 @@ "--dir", "assets", "--version", - "\u003cversion\u003e", + "", "--scope", "native" ] @@ -374,7 +374,7 @@ [ "slsa-verifier", "verify-artifact", - "\u003carchive\u003e", + "", "--provenance-path", "multiple.intoto.jsonl", "--source-uri", @@ -759,8 +759,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", - "sha256": "84fb94c8cec1b0399259be84c5d9b252d8f4df3c15b0ac813eca2c765c363ae1" + "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json" } ], "proves": [ @@ -802,8 +801,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", - "sha256": "6fc075beb7e3eca64c9ca8e79c07ff0160e3ff8a3f4284b8df813ef927ce5b9e" + "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json" } ], "proves": [ @@ -823,8 +821,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/engine/pipeline_fallback_test.go", - "sha256": "c8c189a7eeb7ec13e647c33746deac0aae99238be0e57d0cadfdbef0dee5d140" + "location": "internal/engine/pipeline_fallback_test.go" } ], "required_tools": [ @@ -842,8 +839,7 @@ ], "artifacts": [ { - "path": "internal/engine/pipeline_fallback_test.go", - "sha256": "c8c189a7eeb7ec13e647c33746deac0aae99238be0e57d0cadfdbef0dee5d140" + "path": "internal/engine/pipeline_fallback_test.go" } ], "proves": [ @@ -889,8 +885,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-bun.golden.json", - "sha256": "e4df856333b558128b07165cc16d44a12d9981bbd42054ad2912cfb699a563a5" + "path": "test/smoke/testdata/golden/scan-bun.golden.json" } ], "proves": [ @@ -937,8 +932,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-go.golden.json", - "sha256": "267270c11e350f668c4e018c7d36e675c7412b11d37aec5725380fabb391e2a8" + "path": "test/smoke/testdata/golden/scan-go.golden.json" } ], "proves": [ @@ -986,8 +980,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-maven.golden.json", - "sha256": "7093935763fe2735f42a475ad22c6af6284beb352f3bed02f87521a4f48f383d" + "path": "test/smoke/testdata/golden/scan-maven.golden.json" } ], "proves": [ @@ -1034,8 +1027,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-npm.golden.json", - "sha256": "bb19b6686664f1ce4ba3424168a417b46c89e5680e84f996c879b40281451527" + "path": "test/smoke/testdata/golden/scan-npm.golden.json" } ], "proves": [ @@ -1082,8 +1074,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", - "sha256": "eb67f160516425a5828bc546589dc468d50f37525e47e4dc793a0d166bbc4286" + "path": "test/smoke/testdata/golden/scan-pnpm.golden.json" } ], "proves": [ @@ -1129,8 +1120,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", - "sha256": "38a550af6decc30523da724abd78acddb2e7b659df4651eba8313bd9000a9fd9" + "path": "test/smoke/testdata/golden/scan-python-pip.golden.json" } ], "proves": [ @@ -1177,8 +1167,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-yarn.golden.json", - "sha256": "de649973a762167d0d474532773226c284330ee2bcaab4f6fa6dbd26867e34b2" + "path": "test/smoke/testdata/golden/scan-yarn.golden.json" } ], "proves": [ @@ -1198,8 +1187,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/auditors/license/spdx_policy_matrix_test.go", - "sha256": "846bc1cbff37531829124935192034c3cfdc9267c226ccd5144355d0dceec33b" + "location": "internal/auditors/license/spdx_policy_matrix_test.go" } ], "required_tools": [ @@ -1217,8 +1205,7 @@ ], "artifacts": [ { - "path": "internal/auditors/license/spdx_policy_matrix_test.go", - "sha256": "846bc1cbff37531829124935192034c3cfdc9267c226ccd5144355d0dceec33b" + "path": "internal/auditors/license/spdx_policy_matrix_test.go" } ], "proves": [ @@ -1238,8 +1225,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/engine/diff/diff_test.go", - "sha256": "b2f8300336868c2af153d7c77b88ced45955b7b17a0da09c1d3ba08e72a7af73" + "location": "internal/engine/diff/diff_test.go" } ], "required_tools": [ @@ -1257,8 +1243,7 @@ ], "artifacts": [ { - "path": "internal/engine/diff/diff_test.go", - "sha256": "b2f8300336868c2af153d7c77b88ced45955b7b17a0da09c1d3ba08e72a7af73" + "path": "internal/engine/diff/diff_test.go" } ], "proves": [ @@ -1305,8 +1290,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", - "sha256": "9d70ea49a7c99e9dc2c2bee0ad4c1dc0368bbe02167386d62eb8bb793f39b5ae" + "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json" } ], "proves": [ @@ -1354,8 +1338,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", - "sha256": "60060bf892235feef6df1e2d18e69cd1a00b6a277e8d3537a4dad4fed0a6c022" + "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json" } ], "proves": [ @@ -1402,8 +1385,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", - "sha256": "f9774f51e9e9c3a931a900c9be62b93b5810cf023f500ebf9a96f6f1fdeb2a66" + "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json" } ], "proves": [ @@ -1449,8 +1431,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", - "sha256": "8a40d041d23f4bc4984bb3ba00cc3919592fddcd694b45855e0fb45d5d73077b" + "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json" } ], "proves": [ @@ -1470,8 +1451,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/remediation/derive_test.go", - "sha256": "0be22480c9395c834dbcc23f015629cc837a51d5e122eaac36b520b43d1759ed" + "location": "internal/remediation/derive_test.go" } ], "required_tools": [ @@ -1489,8 +1469,7 @@ ], "artifacts": [ { - "path": "internal/remediation/derive_test.go", - "sha256": "0be22480c9395c834dbcc23f015629cc837a51d5e122eaac36b520b43d1759ed" + "path": "internal/remediation/derive_test.go" } ], "proves": [ @@ -1511,8 +1490,7 @@ "inputs": [ { "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.cdx.json", - "sha256": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" + "location": "test/smoke/testdata/sboms/go.cdx.json" } ], "required_tools": [ @@ -1535,8 +1513,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", - "sha256": "96a1a56860eb1c28bcb0da4d01809f74a7b07e6d0bcf637f2b8e36c6ecd9208e" + "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json" } ], "proves": [ @@ -1557,8 +1534,7 @@ "inputs": [ { "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.spdx.json", - "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + "location": "test/smoke/testdata/sboms/go.spdx.json" } ], "required_tools": [ @@ -1581,8 +1557,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", - "sha256": "a0bb88cb6bbdbbe49e02ca6e091d15953be4320dd82c1dedcff32085dfbf042a" + "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json" } ], "proves": [ @@ -1635,8 +1610,7 @@ ], "artifacts": [ { - "path": "test/smoke/audit_test.go", - "sha256": "d0d49980d6d4a5fd3cc67ac1ca39d5aa4f17bd378faac7831f11f02d3271a1a6" + "path": "test/smoke/audit_test.go" } ], "proves": [ @@ -1656,8 +1630,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "3bc30277032a8a1159e5f1a5d9b4697c0be914368a7946d69a2480754937283b" + "location": "internal/auditors/vulnerability/policy_matrix_test.go" } ], "required_tools": [ @@ -1675,8 +1648,7 @@ ], "artifacts": [ { - "path": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "3bc30277032a8a1159e5f1a5d9b4697c0be914368a7946d69a2480754937283b" + "path": "internal/auditors/vulnerability/policy_matrix_test.go" } ], "proves": [ diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go index f6e346fe..7c28ca0d 100644 --- a/internal/assurance/catalog.go +++ b/internal/assurance/catalog.go @@ -1,8 +1,6 @@ package assurance import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "os" @@ -103,13 +101,20 @@ type Input struct { Location string `json:"location"` Ref string `json:"ref,omitempty"` Revision string `json:"revision,omitempty"` - SHA256 string `json:"sha256,omitempty"` } // EvidenceArtifact is a repository file that records an evidence claim's result. +// +// It is named by path and nothing else. A file in this repository is already +// content-addressed by git: a release tag and a path identify exact bytes, and +// the report records the commit. A checksum stored beside the path would be a +// second, hand-kept copy of what git knows, and the only thing it could ever +// do is fall behind the file -- which it did, on every golden update, until +// it was removed. What cannot be addressed by this repository's history is +// still pinned where it is named: a git input by revision, a container by +// digest. type EvidenceArtifact struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` + Path string `json:"path"` } // Evidence is one public claim about Bomly's behavior, proven by a pinned input @@ -133,7 +138,7 @@ type Evidence struct { } // ParseCatalog decodes and structurally validates a catalog document. It does -// not touch the filesystem; use VerifyArtifacts for the repository-side checks. +// not touch the filesystem; use VerifyFiles for the repository-side checks. func ParseCatalog(data []byte) (Catalog, error) { if len(data) > MaxCatalogBytes { return Catalog{}, fmt.Errorf("catalog is %d bytes, limit is %d", len(data), MaxCatalogBytes) @@ -299,12 +304,9 @@ func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[s return err } if len(evidence.Artifacts) == 0 { - return errCatalog("evidence requires at least one hashed artifact") + return errCatalog("evidence requires at least one artifact") } for _, artifact := range evidence.Artifacts { - if !hashPattern.MatchString(artifact.SHA256) { - return fmt.Errorf("artifact %q has an invalid SHA-256 hash", artifact.Path) - } if err := validateRepoPath(artifact.Path); err != nil { return err } @@ -331,9 +333,6 @@ func validateInput(level EvidenceLevel, input Input) error { return errCatalog("git input requires a full lowercase commit revision") } case "fixture": - if !hashPattern.MatchString(input.SHA256) { - return errCatalog("fixture input requires a SHA-256 hash") - } return validateRepoPath(input.Location) case "container": if input.Ref == "" { @@ -382,118 +381,65 @@ func validateRepoPath(path string) error { return nil } -// RefreshArtifacts rewrites every recorded hash from the file it names and -// reports how many changed. Golden files are regenerated by their own tooling, -// so the catalog has to be able to catch up without anyone editing hashes by -// hand. -func (c *Catalog) RefreshArtifacts(root string) (int, error) { - changed := 0 - for evidenceIndex := range c.Evidence { - evidence := &c.Evidence[evidenceIndex] - for artifactIndex := range evidence.Artifacts { - artifact := &evidence.Artifacts[artifactIndex] - actual, err := hashRepositoryFile(root, artifact.Path) - if err != nil { - return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) - } - if actual != artifact.SHA256 { - artifact.SHA256 = actual - changed++ - } - } - for inputIndex := range evidence.Inputs { - input := &evidence.Inputs[inputIndex] - if input.Kind != "fixture" { - continue - } - actual, err := hashRepositoryFile(root, input.Location) - if err != nil { - return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) - } - if actual != input.SHA256 { - input.SHA256 = actual - changed++ - } - } - } - return changed, nil -} - -// Encode renders the catalog as indented JSON with a trailing newline. -func (c Catalog) Encode() ([]byte, error) { - data, err := json.MarshalIndent(c, "", " ") - if err != nil { - return nil, fmt.Errorf("encode assurance catalog: %w", err) - } - return append(data, '\n'), nil -} - -// VerifyArtifacts confirms every catalog artifact and fixture input still -// matches its recorded hash inside root. -func (c Catalog) VerifyArtifacts(root string) error { +// VerifyFiles confirms every file the catalog names -- each claim's artifacts +// and fixture inputs -- exists inside root as a regular file, and returns how +// many it checked. A claim that points at a file that was renamed or deleted +// would otherwise keep reading as proven. +// +// The count is returned so a caller can refuse a run that reached nothing +// (ADR-0044): a catalog with no files to check and a catalog whose files are +// all present look the same from a nil error. +func (c Catalog) VerifyFiles(root string) (int, error) { + checked := 0 for _, evidence := range c.Evidence { for _, artifact := range evidence.Artifacts { - if err := verifyHashedFile(root, artifact.Path, artifact.SHA256); err != nil { - return fmt.Errorf("evidence %q: %w", evidence.ID, err) + if err := requireRepositoryFile(root, artifact.Path); err != nil { + return checked, fmt.Errorf("evidence %q: %w", evidence.ID, err) } + checked++ } for _, input := range evidence.Inputs { if input.Kind != "fixture" { continue } - if err := verifyHashedFile(root, input.Location, input.SHA256); err != nil { - return fmt.Errorf("evidence %q: %w", evidence.ID, err) + if err := requireRepositoryFile(root, input.Location); err != nil { + return checked, fmt.Errorf("evidence %q: %w", evidence.ID, err) } + checked++ } } - return nil + return checked, nil } -func verifyHashedFile(root, path, want string) error { - actual, err := hashRepositoryFile(root, path) - if err != nil { - return err - } - if actual != want { - return fmt.Errorf("%q hash is %s, want %s", path, actual, want) - } - return nil -} - -// hashRepositoryFile reads one repository file and returns its SHA-256, after -// confirming the path stays inside root. -func hashRepositoryFile(root, path string) (string, error) { +// requireRepositoryFile confirms path names a regular file that stays inside +// root once symlinks are resolved. +func requireRepositoryFile(root, path string) error { if err := validateRepoPath(path); err != nil { - return "", err + return err } resolvedRoot, err := filepath.EvalSymlinks(root) if err != nil { - return "", fmt.Errorf("resolve repository root: %w", err) + return fmt.Errorf("resolve repository root: %w", err) } resolved, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(path))) if err != nil { - return "", fmt.Errorf("resolve %q: %w", path, err) + return fmt.Errorf("resolve %q: %w", path, err) } relative, err := filepath.Rel(resolvedRoot, resolved) if err != nil { - return "", fmt.Errorf("resolve %q relative to repository: %w", path, err) + return fmt.Errorf("resolve %q relative to repository: %w", path, err) } if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("path %q resolves outside the repository", path) + return fmt.Errorf("path %q resolves outside the repository", path) } info, err := os.Stat(resolved) if err != nil { - return "", fmt.Errorf("inspect %q: %w", path, err) + return fmt.Errorf("inspect %q: %w", path, err) } if !info.Mode().IsRegular() { - return "", fmt.Errorf("%q is not a regular file", path) + return fmt.Errorf("%q is not a regular file", path) } - data, err := os.ReadFile(resolved) - if err != nil { - return "", fmt.Errorf("read %q: %w", path, err) - } - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]), nil + return nil } // ChecksForStage returns the catalog checks that belong to one stage. diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go index d2210ce5..cc2fac70 100644 --- a/internal/assurance/catalog_test.go +++ b/internal/assurance/catalog_test.go @@ -23,8 +23,15 @@ func repositoryCatalog(t *testing.T) Catalog { func TestRepositoryCatalogIsValid(t *testing.T) { catalog := repositoryCatalog(t) - if err := catalog.VerifyArtifacts(repositoryRoot); err != nil { - t.Fatalf("catalog artifacts drifted: %v", err) + // Every file a claim names must exist. The count is asserted too: a + // catalog that stopped naming files would otherwise pass this by checking + // nothing (ADR-0044). + checked, err := catalog.VerifyFiles(repositoryRoot) + if err != nil { + t.Fatalf("a claim names a file that is not there: %v", err) + } + if checked < len(catalog.Evidence) { + t.Fatalf("checked %d files for %d evidence claims; every claim names at least one", checked, len(catalog.Evidence)) } for _, stage := range Stages() { if len(catalog.ChecksForStage(stage)) == 0 { @@ -113,7 +120,7 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { }}, Reproduce: [][]string{{"make", "smoke"}}, Artifacts: []EvidenceArtifact{{ - Path: "go.mod", SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + Path: "go.mod", }}, Proves: []string{"It resolves."}, Limitations: []string{"One toolchain."}, @@ -145,13 +152,9 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { c.Evidence[0].Inputs = []Input{{Kind: "workflow", Location: ".github/workflows/smoke.yml"}} }, "evidence without an artifact": func(c *Catalog) { c.Evidence[0].Artifacts = nil }, - "fixture without hash": func(c *Catalog) { - c.Evidence[0].Inputs = []Input{{Kind: "fixture", Location: "go.mod"}} - }, "escaping fixture path": func(c *Catalog) { c.Evidence[0].Inputs = []Input{{ Kind: "fixture", Location: "../../../etc/passwd", - SHA256: "0000000000000000000000000000000000000000000000000000000000000000", }} }, "unsorted checks": func(c *Catalog) { @@ -170,19 +173,50 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { } } -func TestVerifyArtifactsDetectsDrift(t *testing.T) { +func TestVerifyFilesRequiresEveryNamedFile(t *testing.T) { root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { t.Fatalf("write fixture: %v", err) } - catalog := Catalog{Evidence: []Evidence{{ - ID: "example", - Artifacts: []EvidenceArtifact{{ - Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", - }}, - }}} - if err := catalog.VerifyArtifacts(root); err == nil { - t.Fatal("expected a hash mismatch to be reported") + if err := os.Mkdir(filepath.Join(root, "directory"), 0o755); err != nil { + t.Fatalf("create directory: %v", err) + } + claim := func(artifact string, inputs ...Input) Catalog { + return Catalog{Evidence: []Evidence{{ + ID: "example", Artifacts: []EvidenceArtifact{{Path: artifact}}, Inputs: inputs, + }}} + } + + present := claim("golden.json", Input{Kind: "fixture", Location: "golden.json"}, + Input{Kind: "git", Location: "https://example.test/repo"}) + checked, err := present.VerifyFiles(root) + if err != nil { + t.Fatalf("files that exist were rejected: %v", err) + } + // The git input is not a repository file and is not counted. + if checked != 2 { + t.Fatalf("checked %d files, want 2", checked) + } + + // Changing a file's contents is not drift any more: the claim names the + // file, and git records which bytes a release had. + if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{\"changed\":true}\n"), 0o644); err != nil { + t.Fatalf("rewrite fixture: %v", err) + } + if _, err := present.VerifyFiles(root); err != nil { + t.Fatalf("a regenerated file was rejected: %v", err) + } + + for name, catalog := range map[string]Catalog{ + "renamed or deleted artifact": claim("gone.json"), + "artifact is a directory": claim("directory"), + "missing fixture input": claim("golden.json", Input{Kind: "fixture", Location: "gone.json"}), + } { + t.Run(name, func(t *testing.T) { + if _, err := catalog.VerifyFiles(root); err == nil { + t.Fatal("expected the missing file to be reported") + } + }) } } @@ -259,34 +293,3 @@ func TestRepositoryCatalogProducesACompleteReport(t *testing.T) { t.Fatal("the coverage matrix is empty") } } - -func TestRefreshArtifactsRewritesDriftedHashes(t *testing.T) { - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { - t.Fatalf("write fixture: %v", err) - } - catalog := Catalog{Evidence: []Evidence{{ - ID: "example", - Artifacts: []EvidenceArtifact{{ - Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", - }}, - Inputs: []Input{{ - Kind: "fixture", Location: "golden.json", - SHA256: "2222222222222222222222222222222222222222222222222222222222222222", - }}, - }}} - changed, err := catalog.RefreshArtifacts(root) - if err != nil { - t.Fatalf("refresh: %v", err) - } - if changed != 2 { - t.Fatalf("refreshed %d hashes, want 2", changed) - } - if err := catalog.VerifyArtifacts(root); err != nil { - t.Fatalf("refreshed catalog still fails verification: %v", err) - } - again, err := catalog.RefreshArtifacts(root) - if err != nil || again != 0 { - t.Fatalf("second refresh changed %d hashes (err=%v), want 0", again, err) - } -} diff --git a/internal/assurance/cmd/report.go b/internal/assurance/cmd/report.go index e5a857c6..3607b1ee 100644 --- a/internal/assurance/cmd/report.go +++ b/internal/assurance/cmd/report.go @@ -237,8 +237,7 @@ func runCatalogValidate(args []string) error { catalogPath := flags.String("catalog", "", "assurance catalog path") checkID := flags.String("check", "", "print one check") evidenceID := flags.String("evidence", "", "print one evidence claim") - skipArtifacts := flags.Bool("skip-artifacts", false, "skip repository artifact hash verification") - refresh := flags.Bool("refresh", false, "rewrite recorded checksums from the files they name") + skipArtifacts := flags.Bool("skip-artifacts", false, "skip checking that the files the catalog names exist") if err := flags.Parse(args); err != nil { return err } @@ -246,34 +245,19 @@ func runCatalogValidate(args []string) error { if err != nil { return err } - if *refresh { - resolvedRoot, rootErr := repositoryRoot() - if rootErr != nil { - return rootErr - } - changed, refreshErr := catalog.RefreshArtifacts(resolvedRoot) - if refreshErr != nil { - return refreshErr - } - if changed > 0 { - data, encodeErr := catalog.Encode() - if encodeErr != nil { - return encodeErr - } - if err := os.WriteFile(catalogFile, data, 0o644); err != nil { - return fmt.Errorf("write assurance catalog: %w", err) - } - } - fmt.Printf("Refreshed %d recorded checksum(s) in %s.\n", changed, relativeToWorkingDir(catalogFile)) - return nil - } if !*skipArtifacts { resolvedRoot, rootErr := repositoryRoot() if rootErr != nil { return rootErr } - if err := catalog.VerifyArtifacts(resolvedRoot); err != nil { - return err + checked, verifyErr := catalog.VerifyFiles(resolvedRoot) + if verifyErr != nil { + return verifyErr + } + // A run that looked at no files proved nothing about them; say so + // rather than report a clean catalog. + if checked == 0 && len(catalog.Evidence) > 0 { + return fmt.Errorf("the catalog has %d evidence claims but names no files to check", len(catalog.Evidence)) } } fmt.Printf("Validated %s: %d areas, %d checks, %d evidence claims.\n", diff --git a/internal/assurance/testdata/catalog.json b/internal/assurance/testdata/catalog.json index 98993f3d..b68cede1 100644 --- a/internal/assurance/testdata/catalog.json +++ b/internal/assurance/testdata/catalog.json @@ -25,10 +25,22 @@ "stage": "post-release", "level": "advisory", "description": "Runs the same scan several times and records timing.", - "source": { "workflow": "assurance-assessment.yml", "job": "perf-samples" }, - "reproduce": [["make", "benchmark-samples"]], - "proves": ["Repeated runs produced identical normalized output."], - "limitations": ["Timings are observations from one machine, not limits."] + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs produced identical normalized output." + ], + "limitations": [ + "Timings are observations from one machine, not limits." + ] }, { "id": "release-checksums", @@ -37,10 +49,23 @@ "stage": "pre-release", "level": "gate", "description": "Hashes every release file and compares it with SHA256SUMS.", - "source": { "workflow": "release.yml", "job": "verify-draft" }, - "reproduce": [["sha256sum", "--check", "SHA256SUMS"]], - "proves": ["Every published file hashes to the recorded value."], - "limitations": ["Checksums prove the files were not altered after they were built."] + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every published file hashes to the recorded value." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built." + ] }, { "id": "smoke", @@ -49,14 +74,36 @@ "stage": "prerequisites", "level": "gate", "description": "Runs scan against pinned public repositories.", - "source": { "workflow": "smoke.yml", "job": "smoke" }, + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, "expected_instances": [ - { "name": "go", "ecosystems": ["Go"] }, - { "name": "node", "ecosystems": ["JavaScript"] } + { + "name": "go", + "ecosystems": [ + "Go" + ] + }, + { + "name": "node", + "ecosystems": [ + "JavaScript" + ] + } + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "Each ecosystem produces the dependency graph recorded in its golden file." ], - "reproduce": [["make", "smoke"]], - "proves": ["Each ecosystem produces the dependency graph recorded in its golden file."], - "limitations": ["Golden files pin one revision of one example project per ecosystem."] + "limitations": [ + "Golden files pin one revision of one example project per ecosystem." + ] } ], "evidence": [ @@ -76,16 +123,27 @@ "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" } ], - "required_tools": ["git", "go"], - "reproduce": [["make", "smoke"]], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], "artifacts": [ { - "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", - "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json" } ], - "proves": ["The Go detector resolves a build-tool-backed module graph."], - "limitations": ["The result depends on a compatible Go toolchain."] + "proves": [ + "The Go detector resolves a build-tool-backed module graph." + ], + "limitations": [ + "The result depends on a compatible Go toolchain." + ] } ] } diff --git a/internal/assurance/testdata/golden/all-pass.report.json b/internal/assurance/testdata/golden/all-pass.report.json index 6b07becd..11adcf59 100644 --- a/internal/assurance/testdata/golden/all-pass.report.json +++ b/internal/assurance/testdata/golden/all-pass.report.json @@ -342,8 +342,7 @@ ], "artifacts": [ { - "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", - "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json" } ], "proves": [ diff --git a/internal/assurance/testdata/golden/mixed-failure.report.json b/internal/assurance/testdata/golden/mixed-failure.report.json index 7b7aa4e7..b85398e5 100644 --- a/internal/assurance/testdata/golden/mixed-failure.report.json +++ b/internal/assurance/testdata/golden/mixed-failure.report.json @@ -354,8 +354,7 @@ ], "artifacts": [ { - "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", - "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json" } ], "proves": [ From a15887ae8d68228b6dd5adaf820ab215556e1559 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 19:08:30 -0700 Subject: [PATCH 3/8] fix(assurance): report goldens are checked out byte for byte The first prerequisites run failed TestReportGoldens on Windows only. The goldens are compared byte for byte, and a Windows checkout rewrites text files to CRLF; converting a golden to CRLF locally reproduces the same "golden differs" failure. The assurance testdata is marked -text so it is checked out exactly as committed on every platform. Scoped to this directory on purpose: the rest of the repository's testdata has passed on Windows as it is, and changing how it is checked out is not this change's business. Co-Authored-By: Claude Fable 5.1 --- .gitattributes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitattributes b/.gitattributes index a8bcb3ea..18352791 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,6 @@ **/testdata/** export-ignore + +# The assurance goldens and fixtures are compared byte for byte, so they are +# checked out exactly as committed. Without this a Windows checkout rewrites +# their line endings and every golden comparison fails there. +internal/assurance/testdata/** -text From bef6f199ff6ed20929b4b809c00ab61450ada76d Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 19:18:20 -0700 Subject: [PATCH 4/8] fix: three failures the unit suite had only on Windows Nothing ran the unit suite on Windows except the portable stability workflow, which was manual and last ran in July. Making it a release gate ran it again, and three tests had gone red there in the meantime. - SwiftPM published its evidence patterns backslash-separated on Windows. The candidate list was built with filepath.Join and is used both to read files and as the detector's evidence patterns, so the support matrix and `bomly plugin list` said different things depending on the machine. The list is slash-separated now; every read already joins an entry onto a directory with filepath.Join, which converts the separators for the host. - The SBOM detector test looked for a file path inside an error that quotes it. Quoting doubles a Windows path's backslashes, so the bare path was not a substring there. It looks for the quoted path. - The file-cache contract test asserted Unix permission bits. Windows has none: Go reports 0777 for every directory whatever mode it was created with. The mode assertions are Unix-only; the containment half of the contract is still checked everywhere. Co-Authored-By: Claude Fable 5.1 --- internal/detectors/sbom/detector_test.go | 6 +++++- internal/detectors/swiftpm/detector.go | 11 +++++++++-- test/assurance/sdk_contract_test.go | 12 ++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/detectors/sbom/detector_test.go b/internal/detectors/sbom/detector_test.go index 992dfa91..e649b614 100644 --- a/internal/detectors/sbom/detector_test.go +++ b/internal/detectors/sbom/detector_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -167,7 +168,10 @@ func TestDetectorResolveGraph_RejectsSyftJSON(t *testing.T) { if !strings.Contains(err.Error(), "syft convert") { t.Fatalf("ResolveGraph() error %q does not name the conversion", err.Error()) } - if !strings.Contains(err.Error(), path) { + // The error quotes the path, and quoting doubles a Windows path's + // backslashes, so the bare path is not a substring there. Look for the + // path as the message spells it. + if !strings.Contains(err.Error(), strconv.Quote(path)) { t.Fatalf("ResolveGraph() error %q missing the file path", err.Error()) } } diff --git a/internal/detectors/swiftpm/detector.go b/internal/detectors/swiftpm/detector.go index e60730a2..ce0d3169 100644 --- a/internal/detectors/swiftpm/detector.go +++ b/internal/detectors/swiftpm/detector.go @@ -32,11 +32,18 @@ type Detector struct { // evidence detection is built from it, and position attachment walks it. They // had drifted into three different lists, so a project whose only lockfile sat // in the Xcode workspace was recognized for line numbers but not for reading. +// +// The entries are slash-separated on every platform. The list is published as +// the detector's evidence patterns, so it has to read the same in the support +// matrix and in `bomly plugin list` whatever machine produced them; built with +// filepath.Join it came out backslash-separated on Windows. Reading a file +// needs no conversion: every read joins an entry onto a directory with +// filepath.Join, which rewrites the separators for the host. var resolvedCandidates = []string{ "Package.resolved", ".package.resolved", - filepath.Join(".swiftpm", "xcode", "package.xcworkspace", "xcshareddata", "swiftpm", "Package.resolved"), - filepath.Join("project.xcworkspace", "xcshareddata", "swiftpm", "Package.resolved"), + ".swiftpm/xcode/package.xcworkspace/xcshareddata/swiftpm/Package.resolved", + "project.xcworkspace/xcshareddata/swiftpm/Package.resolved", } // evidencePatterns is what makes this detector applicable: any lockfile diff --git a/test/assurance/sdk_contract_test.go b/test/assurance/sdk_contract_test.go index 3c23252f..1a4f20b7 100644 --- a/test/assurance/sdk_contract_test.go +++ b/test/assurance/sdk_contract_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -124,6 +125,13 @@ func TestSDKContractCommandStderrHiddenBelowDebug(t *testing.T) { } } +// unixPermissions reports whether the host has owner/group/other permission +// bits to assert on. Windows does not: access there is governed by ACLs, and +// Go reports every directory as 0777 and every writable file as 0666 whatever +// mode it was created with. The containment half of the cache contract below +// is checked on every platform; only the mode assertions are Unix-only. +const unixPermissions = runtime.GOOS != "windows" + func TestSDKContractFileCachePermissionsAndContainment(t *testing.T) { t.Parallel() @@ -138,7 +146,7 @@ func TestSDKContractFileCachePermissionsAndContainment(t *testing.T) { if err != nil { t.Fatal(err) } - if perm := info.Mode().Perm(); perm != 0o700 { + if perm := info.Mode().Perm(); unixPermissions && perm != 0o700 { t.Fatalf("cache dir permissions = %o, want 700", perm) } @@ -167,7 +175,7 @@ func TestSDKContractFileCachePermissionsAndContainment(t *testing.T) { if err != nil { t.Fatal(err) } - if perm := entryInfo.Mode().Perm(); perm != 0o600 { + if perm := entryInfo.Mode().Perm(); unixPermissions && perm != 0o600 { t.Fatalf("cache entry permissions = %o, want 600", perm) } From e15be860bd7a1e850049538bc8b641be075fab1e Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 19:18:20 -0700 Subject: [PATCH 5/8] perf(assurance): the release stage fuzzes four targets at a time Thirty fuzz targets ran one after another at 45 seconds each, plus a build and a baseline pass per target. That is over half an hour, and it made fuzzing the slowest job in the release prerequisites stage by a wide margin. `scripts/run-fuzz.sh` takes FUZZ_JOBS and runs that many targets at once. Each target is its own `go test` process, so they parallelize cleanly: workers are shared out evenly instead of every fuzzer claiming every CPU, each target's output is printed whole rather than interleaved, and every target is still recorded. The default stays one at a time, so `make fuzz` and the nightly run are unchanged. The prerequisites stage asks for four at a time and thirty seconds each. Run locally with a three-second budget, all thirty targets finish in 77 seconds. A failing target still fails the run and is recorded in both modes. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/assurance-prerequisites.yml | 7 +- .github/workflows/fuzz.yml | 6 ++ scripts/run-fuzz.sh | 71 ++++++++++++++++--- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml index 7a2114c5..a24c5b80 100644 --- a/.github/workflows/assurance-prerequisites.yml +++ b/.github/workflows/assurance-prerequisites.yml @@ -63,7 +63,12 @@ jobs: uses: ./.github/workflows/fuzz.yml with: ref: ${{ inputs.ref || github.ref }} - fuzztime: 45s + # Thirty targets one after another took over half an hour and made this + # the slowest job in the stage. Four at a time for thirty seconds each + # is the same order of total fuzzing in a fraction of the wall clock; + # the nightly run keeps the long budget. + fuzztime: 30s + jobs: 4 assurance: true assurance_tag: ${{ inputs.assurance_tag }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4fe2c268..bf8776d8 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -20,6 +20,11 @@ on: type: string required: false default: 2m + jobs: + description: How many fuzz targets to run at once. + type: number + required: false + default: 1 assurance: description: Upload a release assurance check result. type: boolean @@ -62,6 +67,7 @@ jobs: env: FUZZ_RESULTS_JSONL: ${{ runner.temp }}/fuzz-details.jsonl FUZZTIME: ${{ inputs.fuzztime || '2m' }} + FUZZ_JOBS: ${{ inputs.jobs || 1 }} run: make fuzz FUZZTIME="${FUZZTIME}" - name: Record the fuzz result diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index f20871a7..af55d1e8 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -9,6 +9,54 @@ FUZZTIME="${FUZZTIME:-60s}" # still exits non-zero when any target failed. FUZZ_RESULTS_JSONL="${FUZZ_RESULTS_JSONL:-}" +# FUZZ_JOBS runs that many targets at once (default 1, one after another). +# Each target is its own `go test` process, so they parallelize cleanly; the +# release prerequisites stage uses this to keep thirty targets from taking +# thirty times one target's budget. Every target is attempted in this mode, +# and each one's output is printed whole when it finishes rather than +# interleaved with its neighbours'. +FUZZ_JOBS="${FUZZ_JOBS:-1}" +case "${FUZZ_JOBS}" in + ''|*[!0-9]*|0) echo "FUZZ_JOBS must be a positive integer, got '${FUZZ_JOBS}'" >&2; exit 2 ;; +esac + +# run_target fuzzes one " " pair, records it when a results +# file is set, and returns the target's exit status. +run_target() { + local package="${1%% *}" fuzz="${1#* }" started status=0 + local -a workers=() + # Concurrent fuzzers would each start one worker per CPU and fight over + # them; give each an even share instead. (The array is expanded with the + # ${x[@]+...} form below because bash 3.2, which macOS ships, treats an + # empty array as unset under `set -u`.) + if [ "${FUZZ_JOBS}" -gt 1 ]; then + local cpus + cpus="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)" + workers=("-parallel=$(( cpus / FUZZ_JOBS > 0 ? cpus / FUZZ_JOBS : 1 ))") + fi + echo "==> go test ${package} -run=^$ -fuzz=^${fuzz}$ -fuzztime=${FUZZTIME}" + started="$(date -u +%s)" + go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" ${workers[@]+"${workers[@]}"} || status=$? + if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + # One short line per append, so concurrent targets cannot tear each other. + printf '{"name":"%s %s","exit_code":%s,"duration_s":%s}\n' \ + "${package##*/}" "${fuzz}" "${status}" "$(( $(date -u +%s) - started ))" \ + >> "${FUZZ_RESULTS_JSONL}" + fi + return "${status}" +} + +# Internal entry point for the parallel mode: run one target, print its output +# in one piece, and exit with its status. +if [ "${1:-}" = "--target" ]; then + output="$(mktemp)" + status=0 + run_target "$2" > "${output}" 2>&1 || status=$? + cat "${output}" + rm -f "${output}" + exit "${status}" +fi + # The SDK's own fuzz targets (package URL canonicalization, graph/registry # transport JSON) moved with the sdk package to the bomly-sdk repository and # run there. @@ -50,25 +98,26 @@ if [ -n "${FUZZ_RESULTS_JSONL}" ]; then fi failures=0 + +if [ "${FUZZ_JOBS}" -gt 1 ]; then + # xargs keeps going past a failing target and exits non-zero if any failed. + if ! printf '%s\0' "${targets[@]}" | xargs -0 -n 1 -P "${FUZZ_JOBS}" "$0" --target; then + echo "one or more fuzz targets failed" >&2 + exit 1 + fi + exit 0 +fi + for target in "${targets[@]}"; do - package="${target%% *}" - fuzz="${target#* }" - echo "==> go test ${package} -run=^$ -fuzz=^${fuzz}$ -fuzztime=${FUZZTIME}" - started="$(date -u +%s)" status=0 if [ -n "${FUZZ_RESULTS_JSONL}" ]; then - go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" || status=$? + run_target "${target}" || status=$? else - go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" + run_target "${target}" fi if [ "${status}" -ne 0 ]; then failures=$((failures + 1)) fi - if [ -n "${FUZZ_RESULTS_JSONL}" ]; then - printf '{"name":"%s %s","exit_code":%s,"duration_s":%s}\n' \ - "${package##*/}" "${fuzz}" "${status}" "$(( $(date -u +%s) - started ))" \ - >> "${FUZZ_RESULTS_JSONL}" - fi done if [ "${failures}" -ne 0 ]; then From a3e9d1993d611fedd472221b5704c4a1016f54f8 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 21:23:20 -0700 Subject: [PATCH 6/8] docs(assurance): say when a schema version starts to bind, and guard it Review pointed out that removing the per-file checksum from report claims without raising the report's schema version contradicts the rule this framework wrote down for itself: only an optional addition may keep a version. The rule was stated without its starting point. A version names a shape someone already holds, and for the report that begins with the first report committed to docs/assurance/reports -- from then on bomly.dev has mirrored a document of that shape. No report has been published, the site never read the removed field, and its sync does not validate it, so there was no reader to break. Raising the version would have made the first report ever published a "v2" beside a "v1" that never existed. The developer guide now says that in so many words, and the ADR amendment records it. More usefully, TestPublishedReportsParse makes it mechanical: every committed report is read with the current parser, which rejects unknown fields, so once a report exists a change that strands it fails in `make test`. Checked by mutation -- a report in the current shape passes, one carrying the removed field fails, and an index naming a missing report fails. Also marks the sentence in ADR-0047 that the amendment supersedes, so a reader does not have to reconcile the two. Co-Authored-By: Claude Fable 5.1 --- dev-docs/RELEASE_ASSURANCE.md | 11 +++++ ...ance-is-a-catalog-and-a-result-contract.md | 7 +++- internal/assurance/assurance_test.go | 41 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index d33661fb..51287af2 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -210,6 +210,17 @@ versions are a contract with another repository. `TestSchemaVersionsArePinned` fails on any change to these strings, so raising one is always deliberate. +A version names a shape someone already holds. For the report and index that +starts with the first report committed to `docs/assurance/reports/`: from then +on bomly.dev has mirrored a document of that shape and the rules above apply +without exception. Before any report exists there is nothing to stay +compatible with, and the shape can still be corrected under the same version — +which is how the per-file checksums were removed from report claims before the +first release was assessed. `TestPublishedReportsParse` is what holds the line +afterwards: it reads every committed report with the current parser, which +rejects unknown fields, so a shape change that would strand a published report +fails in `make test` and cannot be made by accident. + ## What the automation needs - The Bomly Release app needs **Issues: Read and write** on `bomly-cli` for the diff --git a/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md b/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md index 63a96c12..15bb675e 100644 --- a/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md +++ b/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md @@ -40,7 +40,8 @@ actually download. most prone to drift run before a tag exists. - Workflow files are no longer pinned by checksum in the catalog. Pinning the file proved only that the file had not changed; the per-release check result - proves the workflow ran and what it found. Fixture and expected-result files + proves the workflow ran and what it found. *(The rest of this point is + superseded; see the amendment below.)* Fixture and expected-result files are still checksummed, because a claim about a golden file is only as good as that file — and regenerating goldens therefore has to refresh the catalog (`catalog-validate --refresh`, wired into `Update Smoke Goldens`). @@ -55,7 +56,9 @@ actually download. > covers when the right workflow makes the change. A claim now names its > files by path; what is verified is that they exist. Git inputs stay pinned > by revision and containers by digest, because this repository's history - > cannot address those. + > cannot address those. The report keeps its schema version: no report had + > been published when the field was removed, so no reader held the old + > shape. The versions bind from the first published report onward. - Reports are committed to the default branch rather than attached to the release, because the exhaustive stage runs after publication and GitHub releases are immutable once published. diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index b213235d..c488efe9 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -497,3 +497,44 @@ func TestParseReportRequiresClaimDescriptions(t *testing.T) { t.Fatal("expected a claim with a blank description to be rejected") } } + +// TestPublishedReportsParse is what makes the report's schema version mean +// something. Every report committed under docs/assurance/reports was mirrored +// by bomly.dev in the shape it had when it was published, so it has to keep +// parsing under the version it declares. The parser rejects unknown fields, so +// removing or renaming a field while reports of that version exist fails here, +// and the only ways forward are to keep the field or raise the version. +// +// Before the first release is assessed there are no reports and nothing to +// protect; the index is what says how many there should be, so a reports +// directory that was emptied or moved is not mistaken for that case. +func TestPublishedReportsParse(t *testing.T) { + directory := filepath.Join(repositoryRoot, "docs", "assurance") + indexData, err := os.ReadFile(filepath.Join(directory, "index.json")) + if os.IsNotExist(err) { + entries, _ := filepath.Glob(filepath.Join(directory, "reports", "*.json")) + if len(entries) > 0 { + t.Fatalf("%d reports are committed but there is no index.json", len(entries)) + } + return + } + if err != nil { + t.Fatalf("read index: %v", err) + } + index, err := ParseIndex(indexData) + if err != nil { + t.Fatalf("the committed index no longer parses: %v", err) + } + if len(index.Releases) == 0 { + t.Fatal("index.json exists but lists no releases") + } + for _, release := range index.Releases { + data, err := os.ReadFile(filepath.Join(directory, "reports", release.Tag+".json")) + if err != nil { + t.Fatalf("the index lists %s but its report cannot be read: %v", release.Tag, err) + } + if _, err := ParseReport(data); err != nil { + t.Fatalf("published report %s no longer parses, so its schema version was broken: %v", release.Tag, err) + } + } +} From f21496903e09ff7b9bb9cb6c06fe419a35259ff3 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 21:31:43 -0700 Subject: [PATCH 7/8] test(assurance): published reports must round-trip byte for byte The guard added in the previous commit had two holes, both found in review. It parsed only the reports the index lists, so a report file the index did not mention was never read. And it relied on the strict parser alone, which rejects a removed or renamed field but not a newly required one: such a field decodes silently as its zero value, so an incompatible addition under the same schema version passed. Both close on one property. Every file in docs/assurance/reports must parse under the current schema and encode back to exactly the bytes it was read from. A removal or rename fails the parser. A newly required field appears in the re-encoding and the bytes differ. An added optional field is omitted when empty, so it passes -- which is precisely the one change allowed to keep a version. The check asks the encoder instead of listing fields, so it cannot fall behind the struct. The directory and the index must also agree in both directions. The same property is asserted on the fixture reports, so it is exercised before the first report is published. Checked by mutation: a current-shape report passes; a field added without omitempty fails; the same field with omitempty passes; an unlisted report fails, and is still read when its shape is broken; a listed report that is missing fails. Co-Authored-By: Claude Fable 5.1 --- dev-docs/RELEASE_ASSURANCE.md | 12 ++-- internal/assurance/assurance_test.go | 85 +++++++++++++++++++++------- 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 51287af2..49e0b60c 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -216,10 +216,14 @@ on bomly.dev has mirrored a document of that shape and the rules above apply without exception. Before any report exists there is nothing to stay compatible with, and the shape can still be corrected under the same version — which is how the per-file checksums were removed from report claims before the -first release was assessed. `TestPublishedReportsParse` is what holds the line -afterwards: it reads every committed report with the current parser, which -rejects unknown fields, so a shape change that would strand a published report -fails in `make test` and cannot be made by accident. +first release was assessed. `TestPublishedReportsKeepTheirShape` is what holds +the line afterwards. It reads every file in `docs/assurance/reports/` and +requires each one to parse under the current schema and encode back to exactly +the bytes it was read from. A removed or renamed field fails the strict parser; +a newly required field parses as a zero value but shows up in the re-encoding; +an added optional field is omitted when empty and so passes, which is the one +change allowed to keep a version. The directory and the index must also agree +in both directions. ## What the automation needs diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index c488efe9..00e5bb1c 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -255,10 +255,9 @@ func TestReportGoldens(t *testing.T) { markdown := RenderMarkdown(report, MarkdownOptions{IncludeChecks: true, IncludeTrends: true}) compareGolden(t, testCase.name+".summary.md", []byte(markdown)) - // The report must survive a round trip through its own parser. - if _, err := ParseReport(encoded); err != nil { - t.Fatalf("parse encoded report: %v", err) - } + // The same property published reports are held to, checked here + // so it is exercised before the first report exists. + requireStableShape(t, testCase.name+".report.json", encoded) }) } } @@ -498,23 +497,62 @@ func TestParseReportRequiresClaimDescriptions(t *testing.T) { } } -// TestPublishedReportsParse is what makes the report's schema version mean -// something. Every report committed under docs/assurance/reports was mirrored -// by bomly.dev in the shape it had when it was published, so it has to keep -// parsing under the version it declares. The parser rejects unknown fields, so -// removing or renaming a field while reports of that version exist fails here, -// and the only ways forward are to keep the field or raise the version. +// requireStableShape fails unless a report document parses under the current +// schema and encodes back to exactly the bytes it was read from. +// +// That one property is the whole compatibility rule. A removed or renamed +// field makes the strict parser reject the document. A newly required field +// parses silently, as its zero value, but then appears in the re-encoding and +// the bytes differ. An added optional field is omitted when empty, so the +// bytes match -- which is exactly the one change allowed to keep a version. +// It asks the encoder rather than listing fields, so it cannot fall behind +// the struct (ADR-0044). +func requireStableShape(t *testing.T, name string, data []byte) { + t.Helper() + report, err := ParseReport(data) + if err != nil { + t.Fatalf("%s no longer parses under the schema version it declares: %v", name, err) + } + encoded, err := report.Encode() + if err != nil { + t.Fatalf("%s: encode: %v", name, err) + } + if string(encoded) != string(data) { + t.Fatalf("%s does not survive a round trip through the current report shape; "+ + "a field was added as required, or its encoding changed, without raising the schema version", name) + } +} + +// TestPublishedReportsKeepTheirShape is what makes the report's schema version +// mean something. Every report committed under docs/assurance/reports was +// mirrored by bomly.dev in the shape it had when it was published, so each one +// has to keep that shape under the version it declares. // -// Before the first release is assessed there are no reports and nothing to -// protect; the index is what says how many there should be, so a reports -// directory that was emptied or moved is not mistaken for that case. -func TestPublishedReportsParse(t *testing.T) { +// Every file in the directory is read, and the directory and the index have to +// agree in both directions: a report the index does not list is still checked +// and is reported, and a listed report that is missing fails. Before the first +// release is assessed there are no reports and no index, and nothing to +// protect. +func TestPublishedReportsKeepTheirShape(t *testing.T) { directory := filepath.Join(repositoryRoot, "docs", "assurance") + files, err := filepath.Glob(filepath.Join(directory, "reports", "*.json")) + if err != nil { + t.Fatalf("list reports: %v", err) + } + committed := map[string]struct{}{} + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + requireStableShape(t, filepath.Base(file), data) + committed[strings.TrimSuffix(filepath.Base(file), ".json")] = struct{}{} + } + indexData, err := os.ReadFile(filepath.Join(directory, "index.json")) if os.IsNotExist(err) { - entries, _ := filepath.Glob(filepath.Join(directory, "reports", "*.json")) - if len(entries) > 0 { - t.Fatalf("%d reports are committed but there is no index.json", len(entries)) + if len(committed) > 0 { + t.Fatalf("%d reports are committed but there is no index.json", len(committed)) } return } @@ -528,13 +566,16 @@ func TestPublishedReportsParse(t *testing.T) { if len(index.Releases) == 0 { t.Fatal("index.json exists but lists no releases") } + listed := map[string]struct{}{} for _, release := range index.Releases { - data, err := os.ReadFile(filepath.Join(directory, "reports", release.Tag+".json")) - if err != nil { - t.Fatalf("the index lists %s but its report cannot be read: %v", release.Tag, err) + listed[release.Tag] = struct{}{} + if _, ok := committed[release.Tag]; !ok { + t.Errorf("the index lists %s but reports/%s.json is not committed", release.Tag, release.Tag) } - if _, err := ParseReport(data); err != nil { - t.Fatalf("published report %s no longer parses, so its schema version was broken: %v", release.Tag, err) + } + for tag := range committed { + if _, ok := listed[tag]; !ok { + t.Errorf("reports/%s.json is committed but the index does not list it", tag) } } } From 5fa65b6145b203b09835625dba28d1baeebf0020 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Sat, 3 Oct 2026 21:40:24 -0700 Subject: [PATCH 8/8] test(assurance): remove the published-report shape guard This pull request fixes a stale-checksum failure. In its first review round a reviewer noted that dropping a report field without raising the schema version contradicted the written rule, so a guard was added to hold that rule mechanically. Every round since has been about the guard: - round two: it skipped report files the index did not list, and it let a newly required field through, because such a field decodes as a zero value; - round three: it did not cover the index, and its byte comparison would fail on a Windows checkout, which rewrites line endings. The last one is the reason to stop. That failure would land in the portable suite, which gates a release, as soon as the first real report was committed. A guard that byte-compares published data had become a release risk in its own right, and it was being refined in place of the change it was attached to. So it is removed, and the reasoning is recorded where the next person will look: beside TestSchemaVersionsArePinned and in the developer guide. The rule itself is unchanged and is still stated there, including when a version starts to bind. It is held by the pinned-version test, by the report goldens, where any change of shape shows up as a diff in review, and by the reviewer. A mechanical guard, if wanted, is its own change, and should compare the report's JSON schema against the one recorded at the first published release rather than committed bytes. Co-Authored-By: Claude Fable 5.1 --- dev-docs/RELEASE_ASSURANCE.md | 20 +++--- internal/assurance/assurance_test.go | 102 +++++---------------------- 2 files changed, 28 insertions(+), 94 deletions(-) diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 49e0b60c..245f6cb6 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -216,14 +216,18 @@ on bomly.dev has mirrored a document of that shape and the rules above apply without exception. Before any report exists there is nothing to stay compatible with, and the shape can still be corrected under the same version — which is how the per-file checksums were removed from report claims before the -first release was assessed. `TestPublishedReportsKeepTheirShape` is what holds -the line afterwards. It reads every file in `docs/assurance/reports/` and -requires each one to parse under the current schema and encode back to exactly -the bytes it was read from. A removed or renamed field fails the strict parser; -a newly required field parses as a zero value but shows up in the re-encoding; -an added optional field is omitted when empty and so passes, which is the one -change allowed to keep a version. The directory and the index must also agree -in both directions. +first release was assessed. + +Nothing checks committed reports against the current shape automatically, and +that is deliberate. A test that did so was written and removed in the same +pull request: each review round found another gap in it, and comparing +published files byte for byte would have failed on a Windows checkout — in the +portable suite, which gates a release. The rule is held by +`TestSchemaVersionsArePinned`, by the report goldens (any change to the shape +shows up as a golden diff in review), and by the reviewer. If a mechanical +guard is wanted later, build it as its own change and compare the report's +JSON schema with the one recorded at the first published release, rather than +comparing committed bytes. ## What the automation needs diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index 00e5bb1c..64d7a019 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -255,9 +255,10 @@ func TestReportGoldens(t *testing.T) { markdown := RenderMarkdown(report, MarkdownOptions{IncludeChecks: true, IncludeTrends: true}) compareGolden(t, testCase.name+".summary.md", []byte(markdown)) - // The same property published reports are held to, checked here - // so it is exercised before the first report exists. - requireStableShape(t, testCase.name+".report.json", encoded) + // The report must survive a round trip through its own parser. + if _, err := ParseReport(encoded); err != nil { + t.Fatalf("parse encoded report: %v", err) + } }) } } @@ -416,6 +417,18 @@ func TestMatchJobURLPicksTheRunningJobOnThisRunner(t *testing.T) { // published reports are read by bomly.dev, which renders a known list of // versions: adding an optional field keeps the version, while removing or // repurposing one has to raise it and be taught to the site first. +// +// There is deliberately no test that checks committed reports against the +// current shape. One was written and removed in the same pull request (#492): +// three review rounds each found another hole in it -- files the index did not +// list, required fields that decode as zero values, the index itself, and line +// endings on a Windows checkout -- and the last of those would have failed the +// portable suite and blocked releases as soon as a real report was committed. +// A guard that byte-compares published data is a release risk of its own, and +// it was being refined instead of the change it was attached to. The rule is +// held by this test, the report goldens, and review. If it needs a mechanical +// guard, that is its own change: compare the report's JSON schema against the +// one recorded at the first published release, not committed bytes. func TestSchemaVersionsArePinned(t *testing.T) { for name, actual := range map[string]string{ "check": CheckSchema, @@ -496,86 +509,3 @@ func TestParseReportRequiresClaimDescriptions(t *testing.T) { t.Fatal("expected a claim with a blank description to be rejected") } } - -// requireStableShape fails unless a report document parses under the current -// schema and encodes back to exactly the bytes it was read from. -// -// That one property is the whole compatibility rule. A removed or renamed -// field makes the strict parser reject the document. A newly required field -// parses silently, as its zero value, but then appears in the re-encoding and -// the bytes differ. An added optional field is omitted when empty, so the -// bytes match -- which is exactly the one change allowed to keep a version. -// It asks the encoder rather than listing fields, so it cannot fall behind -// the struct (ADR-0044). -func requireStableShape(t *testing.T, name string, data []byte) { - t.Helper() - report, err := ParseReport(data) - if err != nil { - t.Fatalf("%s no longer parses under the schema version it declares: %v", name, err) - } - encoded, err := report.Encode() - if err != nil { - t.Fatalf("%s: encode: %v", name, err) - } - if string(encoded) != string(data) { - t.Fatalf("%s does not survive a round trip through the current report shape; "+ - "a field was added as required, or its encoding changed, without raising the schema version", name) - } -} - -// TestPublishedReportsKeepTheirShape is what makes the report's schema version -// mean something. Every report committed under docs/assurance/reports was -// mirrored by bomly.dev in the shape it had when it was published, so each one -// has to keep that shape under the version it declares. -// -// Every file in the directory is read, and the directory and the index have to -// agree in both directions: a report the index does not list is still checked -// and is reported, and a listed report that is missing fails. Before the first -// release is assessed there are no reports and no index, and nothing to -// protect. -func TestPublishedReportsKeepTheirShape(t *testing.T) { - directory := filepath.Join(repositoryRoot, "docs", "assurance") - files, err := filepath.Glob(filepath.Join(directory, "reports", "*.json")) - if err != nil { - t.Fatalf("list reports: %v", err) - } - committed := map[string]struct{}{} - for _, file := range files { - data, err := os.ReadFile(file) - if err != nil { - t.Fatalf("read %s: %v", file, err) - } - requireStableShape(t, filepath.Base(file), data) - committed[strings.TrimSuffix(filepath.Base(file), ".json")] = struct{}{} - } - - indexData, err := os.ReadFile(filepath.Join(directory, "index.json")) - if os.IsNotExist(err) { - if len(committed) > 0 { - t.Fatalf("%d reports are committed but there is no index.json", len(committed)) - } - return - } - if err != nil { - t.Fatalf("read index: %v", err) - } - index, err := ParseIndex(indexData) - if err != nil { - t.Fatalf("the committed index no longer parses: %v", err) - } - if len(index.Releases) == 0 { - t.Fatal("index.json exists but lists no releases") - } - listed := map[string]struct{}{} - for _, release := range index.Releases { - listed[release.Tag] = struct{}{} - if _, ok := committed[release.Tag]; !ok { - t.Errorf("the index lists %s but reports/%s.json is not committed", release.Tag, release.Tag) - } - } - for tag := range committed { - if _, ok := listed[tag]; !ok { - t.Errorf("reports/%s.json is committed but the index does not list it", tag) - } - } -}