diff --git a/.gitattributes b/.gitattributes index a8bcb3ea..18352791 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,6 @@ **/testdata/** export-ignore + +# The assurance goldens and fixtures are compared byte for byte, so they are +# checked out exactly as committed. Without this a Windows checkout rewrites +# their line endings and every golden comparison fails there. +internal/assurance/testdata/** -text diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml index af64c556..a24c5b80 100644 --- a/.github/workflows/assurance-prerequisites.yml +++ b/.github/workflows/assurance-prerequisites.yml @@ -63,7 +63,12 @@ jobs: uses: ./.github/workflows/fuzz.yml with: ref: ${{ inputs.ref || github.ref }} - fuzztime: 45s + # Thirty targets one after another took over half an hour and made this + # the slowest job in the stage. Four at a time for thirty seconds each + # is the same order of total fuzzing in a fraction of the wall clock; + # the nightly run keeps the long budget. + fuzztime: 30s + jobs: 4 assurance: true assurance_tag: ${{ inputs.assurance_tag }} @@ -98,10 +103,10 @@ jobs: run: | set -euo pipefail exit_code=0 - summary="The assurance catalog is valid and every fixture and expected-result file it names still matches its recorded checksum." + summary="The assurance catalog is valid and every fixture and expected-result file it names exists." if [ "${CATALOG_OUTCOME}" != "success" ]; then exit_code=1 - summary="The assurance catalog is invalid or one of the files it names has changed. Open the validation step for the exact entry." + summary="The assurance catalog is invalid or names a file that is not there. Open the validation step for the exact entry." fi checks="$(grep -c '"stage":' docs/assurance/catalog.json || true)" # --stage and --level are explicit: when the catalog is the thing diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4fe2c268..bf8776d8 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -20,6 +20,11 @@ on: type: string required: false default: 2m + jobs: + description: How many fuzz targets to run at once. + type: number + required: false + default: 1 assurance: description: Upload a release assurance check result. type: boolean @@ -62,6 +67,7 @@ jobs: env: FUZZ_RESULTS_JSONL: ${{ runner.temp }}/fuzz-details.jsonl FUZZTIME: ${{ inputs.fuzztime || '2m' }} + FUZZ_JOBS: ${{ inputs.jobs || 1 }} run: make fuzz FUZZTIME="${FUZZTIME}" - name: Record the fuzz result diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index e39764bd..f536b8eb 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -319,14 +319,6 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: go.mod - # No cache writes: this job runs with write permissions, and a - # poisoned cache would reach the jobs that trust it. - cache: false - - name: Download regenerated goldens uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -357,16 +349,6 @@ jobs: --jq '[.jobs[] | select(.name | startswith("Regenerate (")) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out") | .name] | join(", ")')" echo "failed=${failed}" >> "$GITHUB_OUTPUT" - - name: Refresh the catalog checksums - # Public evidence claims record the checksum of the golden file that - # backs them. Regenerating a golden without refreshing the catalog - # would fail the catalog-valid gate and block the next release. - if: steps.changes.outputs.changed == 'true' - shell: bash - run: | - set -euo pipefail - go run ./internal/assurance/cmd catalog-validate --refresh - - name: Create pull request with updated goldens if: steps.changes.outputs.changed == 'true' env: @@ -387,7 +369,7 @@ jobs: git checkout -b "${branch_name}" - git add test/smoke/testdata/golden docs/assurance/catalog.json + git add test/smoke/testdata/golden git commit -m "test: update smoke golden files" git push --set-upstream origin "${branch_name}" diff --git a/AGENTS.md b/AGENTS.md index 3800d1f8..56f8d30a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -354,7 +354,7 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai - A golden must never carry a host architecture. `make smoke ARGS="-update"` on an arm64 laptop writes a golden CI cannot match, so `normalizeJSON` erases the architecture and `TestGoldensCarryNoHostArchitecture` (untagged, runs in `make test`) fails if one reaches a committed golden. A new architecture-bearing shape means teaching the normalizer, not excepting the guard. - Register new tests in both slice matrices (`smoke.yml` and exactly one slice in `update-smoke-goldens.yml`); `go test -run` elements are unanchored regexes — use `$` anchors to keep slice ownership exact. - A new slice also needs an entry in the `smoke` check's `expected_instances` in `docs/assurance/catalog.json` (with its `ecosystems`, which is what puts an ecosystem on the report's coverage list). `TestCatalogSmokeInstancesMatchWorkflowMatrix` fails when the two drift. -- Regenerating goldens invalidates the checksums the catalog's claims pin. `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the catalog with them; do the same when refreshing by hand. +- Regenerating goldens needs no catalog change. A claim names its expected-result file by path only; git already records which bytes a release had, so there is no stored checksum to fall behind. Renaming or deleting a file a claim names does fail `TestRepositoryCatalogIsValid` in `make test`. - `TestExamplePluginFixtureCompiles` runs in `make test` and must keep compiling against the pinned `bomly-dev/bomly-sdk` release; update the fixture source when the SDK contract changes. ## Release assurance @@ -370,7 +370,7 @@ Rules: - Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. - Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. - `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. -- Public evidence claims live in the same catalog (`evidence[]`), keep their pinned Git revisions and checksummed artifacts, and name the check that backs them. `make assurance-catalog` re-hashes every file they reference. +- Public evidence claims live in the same catalog (`evidence[]`), pin what this repository's history cannot address (a Git input by revision, a container by digest), name their fixture and expected-result files by path, and name the check that backs them. `make assurance-catalog` confirms every file they name exists. Do not add a checksum of an in-repository file to the catalog: it is a second copy of what git knows, and it can only drift. - The per-release report (`docs/assurance/reports/.json`) and `docs/assurance/index.json` are written by the post-release assessment and are the only data source for the public page. See [`dev-docs/RELEASE_ASSURANCE.md`](dev-docs/RELEASE_ASSURANCE.md) for the contracts and how to add a check. diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index a6bc0f84..245f6cb6 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -142,11 +142,18 @@ check `missing` — that is the intended behavior: gaps are loud. Evidence claims are the public "we prove X, we do not prove Y" statements that used to live in `test/evidence/cases.json`. They keep the same rigor: pinned -Git revisions, checksummed fixtures and expected-result files, explicit -reproduction commands, and mandatory limitations. `make assurance-catalog` -verifies every hash a claim names, so a golden file cannot drift away from the -claim it supports, and `--refresh` rewrites them when the goldens are -deliberately regenerated. +Git revisions and container digests, explicit reproduction commands, and +mandatory limitations. + +Fixture and expected-result files are named by path, with no checksum. A file +in this repository is already content-addressed by git — a release tag and a +path identify exact bytes, and the report records the commit — so a checksum +stored in the catalog was a second, hand-kept copy of something git knows. It +could only fall behind, and it did on every golden update: two pull requests +that each passed CI left `main` with a catalog that failed its own gate. What +is checked now is what can actually go wrong: `make assurance-catalog`, and +`TestRepositoryCatalogIsValid` in `make test`, fail when a claim names a file +that was renamed or deleted. A claim must carry a pinned input **and** a committed artifact. That is the line between the two layers: if a statement would only restate what its check already @@ -203,6 +210,25 @@ versions are a contract with another repository. `TestSchemaVersionsArePinned` fails on any change to these strings, so raising one is always deliberate. +A version names a shape someone already holds. For the report and index that +starts with the first report committed to `docs/assurance/reports/`: from then +on bomly.dev has mirrored a document of that shape and the rules above apply +without exception. Before any report exists there is nothing to stay +compatible with, and the shape can still be corrected under the same version — +which is how the per-file checksums were removed from report claims before the +first release was assessed. + +Nothing checks committed reports against the current shape automatically, and +that is deliberate. A test that did so was written and removed in the same +pull request: each review round found another gap in it, and comparing +published files byte for byte would have failed on a Windows checkout — in the +portable suite, which gates a release. The rule is held by +`TestSchemaVersionsArePinned`, by the report goldens (any change to the shape +shows up as a golden diff in review), and by the reviewer. If a mechanical +guard is wanted later, build it as its own change and compare the report's +JSON schema with the one recorded at the first published release, rather than +comparing committed bytes. + ## What the automation needs - The Bomly Release app needs **Issues: Read and write** on `bomly-cli` for the @@ -214,11 +240,9 @@ one is always deliberate. `assurance-prerequisites.yml` would miss every stage that Auto Version triggered. Keep that job name stable, or update the lookup in `release.yml` and `assurance-assessment.yml` with it. -- Regenerating smoke goldens changes files the evidence claims are pinned to. - `Update Smoke Goldens` runs `catalog-validate --refresh` and commits the - catalog alongside them; do the same when refreshing goldens by hand. - That workflow regenerates from the ref it was dispatched on, so the refresh - runs the same commit's tooling against that commit's goldens. +- Regenerating smoke goldens needs no catalog change; claims name their + expected-result files by path only. Do not reintroduce a stored checksum of + an in-repository file. ## Related documents diff --git a/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md b/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md index f1b80627..15bb675e 100644 --- a/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md +++ b/dev-docs/adr/0047-release-assurance-is-a-catalog-and-a-result-contract.md @@ -40,10 +40,25 @@ actually download. most prone to drift run before a tag exists. - Workflow files are no longer pinned by checksum in the catalog. Pinning the file proved only that the file had not changed; the per-release check result - proves the workflow ran and what it found. Fixture and expected-result files + proves the workflow ran and what it found. *(The rest of this point is + superseded; see the amendment below.)* Fixture and expected-result files are still checksummed, because a claim about a golden file is only as good as that file — and regenerating goldens therefore has to refresh the catalog (`catalog-validate --refresh`, wired into `Update Smoke Goldens`). + + > **Amended 2026-10-03:** the catalog no longer stores a checksum for any + > file in this repository, and `--refresh` is gone. The argument above for + > workflow files applies to goldens too: git already addresses a file by + > tag and path, so the stored checksum was a hand-kept copy that could only + > fall behind. It did, the first time the prerequisites stage ran on `main` + > — a golden update and the catalog each passed CI separately and failed + > together, which no earlier check could have caught and a refresh step only + > covers when the right workflow makes the change. A claim now names its + > files by path; what is verified is that they exist. Git inputs stay pinned + > by revision and containers by digest, because this repository's history + > cannot address those. The report keeps its schema version: no report had + > been published when the field was removed, so no reader held the old + > shape. The versions bind from the first published report onward. - Reports are committed to the default branch rather than attached to the release, because the exhaustive stage runs after publication and GitHub releases are immutable once published. diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index 7c5dcbf7..7628f7d5 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -79,7 +79,7 @@ The report uses one vocabulary, and it is worth being precise about it: Some claims are asserted directly by a check. Others are asserted by comparing a pinned input — a public repository at a recorded commit, or a checked-in -fixture — against a checksummed result file, and name the check that performed +fixture — against a recorded result file, and name the check that performed that comparison. Both appear as claims on the page, in the section they belong to. A claim earns a separate entry only when it adds something the check cannot say on its own. @@ -94,8 +94,8 @@ make assurance-catalog go run ./internal/assurance/cmd catalog-validate --evidence graph-npm ``` -The first command validates the whole catalog, including the checksums of every -expected-result file it names. The second prints one claim with its reproduction +The first command validates the whole catalog and confirms every expected-result +file it names exists. The second prints one claim with its reproduction command. ## What a verified claim does not mean diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index 2188fa51..455d7857 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -79,7 +79,7 @@ "area": "framework", "stage": "prerequisites", "level": "gate", - "description": "Validates this catalog and confirms every fixture and golden file it points at still has the recorded checksum.", + "description": "Validates this catalog and confirms every fixture and expected-result file it points at exists.", "source": { "workflow": "assurance-prerequisites.yml", "job": "catalog" @@ -91,7 +91,7 @@ ] ], "proves": [ - "Every claim in this report points at a real check and at repository files that have not changed since the claim was written." + "Every claim in this report points at a real check and at files that are present in this release's source." ], "limitations": [ "A valid catalog says the paperwork is consistent; the checks themselves decide whether the software works." @@ -180,7 +180,7 @@ [ "sh", "-c", - "BOMLY_VERSION=\u003ctag\u003e BOMLY_INSTALL_DIR=$PWD/bin sh scripts/install.sh" + "BOMLY_VERSION= BOMLY_INSTALL_DIR=$PWD/bin sh scripts/install.sh" ] ], "proves": [ @@ -230,7 +230,7 @@ "gh", "release", "download", - "\u003ctag\u003e", + "", "--repo", "bomly-dev/bomly-cli" ] @@ -264,7 +264,7 @@ "--dir", "assets", "--version", - "\u003cversion\u003e" + "" ] ], "proves": [ @@ -308,7 +308,7 @@ "--dir", "assets", "--version", - "\u003cversion\u003e", + "", "--scope", "native" ] @@ -374,7 +374,7 @@ [ "slsa-verifier", "verify-artifact", - "\u003carchive\u003e", + "", "--provenance-path", "multiple.intoto.jsonl", "--source-uri", @@ -759,8 +759,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", - "sha256": "0393003fb3d39f8c923325062bbb257606c96a9f60e278cad64c106527ebaef7" + "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json" } ], "proves": [ @@ -802,8 +801,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", - "sha256": "b17ad417464c74d9c8705177d3b067c8d20ab63d382ee4476ad1da9e2748d8a5" + "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json" } ], "proves": [ @@ -823,8 +821,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/engine/pipeline_fallback_test.go", - "sha256": "c8c189a7eeb7ec13e647c33746deac0aae99238be0e57d0cadfdbef0dee5d140" + "location": "internal/engine/pipeline_fallback_test.go" } ], "required_tools": [ @@ -842,8 +839,7 @@ ], "artifacts": [ { - "path": "internal/engine/pipeline_fallback_test.go", - "sha256": "c8c189a7eeb7ec13e647c33746deac0aae99238be0e57d0cadfdbef0dee5d140" + "path": "internal/engine/pipeline_fallback_test.go" } ], "proves": [ @@ -889,8 +885,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-bun.golden.json", - "sha256": "beae880e2ec43d1aa5c0da09bc86903ffc370638ab5b416bf521dfb60d9a8693" + "path": "test/smoke/testdata/golden/scan-bun.golden.json" } ], "proves": [ @@ -937,8 +932,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-go.golden.json", - "sha256": "1352983b2aafc221b9058a82c3edd1e1f8d983b5908ecbcd01a5e0cc11233644" + "path": "test/smoke/testdata/golden/scan-go.golden.json" } ], "proves": [ @@ -986,8 +980,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-maven.golden.json", - "sha256": "96feaa39866e32319b9e0009541afb59f1b856515636ffdc72fb1a5a1b4107f4" + "path": "test/smoke/testdata/golden/scan-maven.golden.json" } ], "proves": [ @@ -1034,8 +1027,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-npm.golden.json", - "sha256": "96f65ace60a7b75087bdad36bda45959aa769ac4dd2c57f13cdc8983b11ed4be" + "path": "test/smoke/testdata/golden/scan-npm.golden.json" } ], "proves": [ @@ -1082,8 +1074,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", - "sha256": "972452883100713d6cc20d1ba5739b548940288571a4bca240b96fa14bcc586e" + "path": "test/smoke/testdata/golden/scan-pnpm.golden.json" } ], "proves": [ @@ -1129,8 +1120,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", - "sha256": "9a5edd634c17c6b04c51135b1c954ae509043f484dadf49c46427322f7bcc9a9" + "path": "test/smoke/testdata/golden/scan-python-pip.golden.json" } ], "proves": [ @@ -1177,8 +1167,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-yarn.golden.json", - "sha256": "8aacecd44516a5cda5a4d84bdfbedb982095924580119d8d7cc8b5f2de6dd57e" + "path": "test/smoke/testdata/golden/scan-yarn.golden.json" } ], "proves": [ @@ -1198,8 +1187,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/auditors/license/spdx_policy_matrix_test.go", - "sha256": "846bc1cbff37531829124935192034c3cfdc9267c226ccd5144355d0dceec33b" + "location": "internal/auditors/license/spdx_policy_matrix_test.go" } ], "required_tools": [ @@ -1217,8 +1205,7 @@ ], "artifacts": [ { - "path": "internal/auditors/license/spdx_policy_matrix_test.go", - "sha256": "846bc1cbff37531829124935192034c3cfdc9267c226ccd5144355d0dceec33b" + "path": "internal/auditors/license/spdx_policy_matrix_test.go" } ], "proves": [ @@ -1238,8 +1225,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/engine/diff/diff_test.go", - "sha256": "b2f8300336868c2af153d7c77b88ced45955b7b17a0da09c1d3ba08e72a7af73" + "location": "internal/engine/diff/diff_test.go" } ], "required_tools": [ @@ -1257,8 +1243,7 @@ ], "artifacts": [ { - "path": "internal/engine/diff/diff_test.go", - "sha256": "b2f8300336868c2af153d7c77b88ced45955b7b17a0da09c1d3ba08e72a7af73" + "path": "internal/engine/diff/diff_test.go" } ], "proves": [ @@ -1305,8 +1290,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", - "sha256": "0e66a3ef13f9555d326f385fac977a910ab2dbe21da5d05affef4ab03e0b7cb6" + "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json" } ], "proves": [ @@ -1354,8 +1338,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", - "sha256": "9e8f69f58dfab4ff9e0e41dec928c57468d7731ad1a981b904f0bf97d2cac5a6" + "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json" } ], "proves": [ @@ -1402,8 +1385,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", - "sha256": "2ea78dc8dc40f0aebf84442300672481c4feaa24c0b5abdf012a7da4c06874fd" + "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json" } ], "proves": [ @@ -1449,8 +1431,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", - "sha256": "181b80b3f2807d6607ccedfa8aafc99b370a78d7161c540316b8a323765be56d" + "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json" } ], "proves": [ @@ -1470,8 +1451,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/remediation/derive_test.go", - "sha256": "0be22480c9395c834dbcc23f015629cc837a51d5e122eaac36b520b43d1759ed" + "location": "internal/remediation/derive_test.go" } ], "required_tools": [ @@ -1489,8 +1469,7 @@ ], "artifacts": [ { - "path": "internal/remediation/derive_test.go", - "sha256": "0be22480c9395c834dbcc23f015629cc837a51d5e122eaac36b520b43d1759ed" + "path": "internal/remediation/derive_test.go" } ], "proves": [ @@ -1511,8 +1490,7 @@ "inputs": [ { "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.cdx.json", - "sha256": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" + "location": "test/smoke/testdata/sboms/go.cdx.json" } ], "required_tools": [ @@ -1535,8 +1513,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", - "sha256": "726d17483521ff675cc8610915e16f701317f36c15f70fe8cb408f1830a3ea2e" + "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json" } ], "proves": [ @@ -1557,8 +1534,7 @@ "inputs": [ { "kind": "fixture", - "location": "test/smoke/testdata/sboms/go.spdx.json", - "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + "location": "test/smoke/testdata/sboms/go.spdx.json" } ], "required_tools": [ @@ -1581,8 +1557,7 @@ ], "artifacts": [ { - "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", - "sha256": "5197ec06425ab1715a6646b230319d25d8eb5409bf08da7a2657e571175d8eee" + "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json" } ], "proves": [ @@ -1635,8 +1610,7 @@ ], "artifacts": [ { - "path": "test/smoke/audit_test.go", - "sha256": "d0d49980d6d4a5fd3cc67ac1ca39d5aa4f17bd378faac7831f11f02d3271a1a6" + "path": "test/smoke/audit_test.go" } ], "proves": [ @@ -1656,8 +1630,7 @@ "inputs": [ { "kind": "fixture", - "location": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "3bc30277032a8a1159e5f1a5d9b4697c0be914368a7946d69a2480754937283b" + "location": "internal/auditors/vulnerability/policy_matrix_test.go" } ], "required_tools": [ @@ -1675,8 +1648,7 @@ ], "artifacts": [ { - "path": "internal/auditors/vulnerability/policy_matrix_test.go", - "sha256": "3bc30277032a8a1159e5f1a5d9b4697c0be914368a7946d69a2480754937283b" + "path": "internal/auditors/vulnerability/policy_matrix_test.go" } ], "proves": [ diff --git a/internal/assurance/assurance_test.go b/internal/assurance/assurance_test.go index b213235d..64d7a019 100644 --- a/internal/assurance/assurance_test.go +++ b/internal/assurance/assurance_test.go @@ -417,6 +417,18 @@ func TestMatchJobURLPicksTheRunningJobOnThisRunner(t *testing.T) { // published reports are read by bomly.dev, which renders a known list of // versions: adding an optional field keeps the version, while removing or // repurposing one has to raise it and be taught to the site first. +// +// There is deliberately no test that checks committed reports against the +// current shape. One was written and removed in the same pull request (#492): +// three review rounds each found another hole in it -- files the index did not +// list, required fields that decode as zero values, the index itself, and line +// endings on a Windows checkout -- and the last of those would have failed the +// portable suite and blocked releases as soon as a real report was committed. +// A guard that byte-compares published data is a release risk of its own, and +// it was being refined instead of the change it was attached to. The rule is +// held by this test, the report goldens, and review. If it needs a mechanical +// guard, that is its own change: compare the report's JSON schema against the +// one recorded at the first published release, not committed bytes. func TestSchemaVersionsArePinned(t *testing.T) { for name, actual := range map[string]string{ "check": CheckSchema, diff --git a/internal/assurance/catalog.go b/internal/assurance/catalog.go index f6e346fe..7c28ca0d 100644 --- a/internal/assurance/catalog.go +++ b/internal/assurance/catalog.go @@ -1,8 +1,6 @@ package assurance import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "os" @@ -103,13 +101,20 @@ type Input struct { Location string `json:"location"` Ref string `json:"ref,omitempty"` Revision string `json:"revision,omitempty"` - SHA256 string `json:"sha256,omitempty"` } // EvidenceArtifact is a repository file that records an evidence claim's result. +// +// It is named by path and nothing else. A file in this repository is already +// content-addressed by git: a release tag and a path identify exact bytes, and +// the report records the commit. A checksum stored beside the path would be a +// second, hand-kept copy of what git knows, and the only thing it could ever +// do is fall behind the file -- which it did, on every golden update, until +// it was removed. What cannot be addressed by this repository's history is +// still pinned where it is named: a git input by revision, a container by +// digest. type EvidenceArtifact struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` + Path string `json:"path"` } // Evidence is one public claim about Bomly's behavior, proven by a pinned input @@ -133,7 +138,7 @@ type Evidence struct { } // ParseCatalog decodes and structurally validates a catalog document. It does -// not touch the filesystem; use VerifyArtifacts for the repository-side checks. +// not touch the filesystem; use VerifyFiles for the repository-side checks. func ParseCatalog(data []byte) (Catalog, error) { if len(data) > MaxCatalogBytes { return Catalog{}, fmt.Errorf("catalog is %d bytes, limit is %d", len(data), MaxCatalogBytes) @@ -299,12 +304,9 @@ func validateEvidence(evidence Evidence, areas map[string]struct{}, checks map[s return err } if len(evidence.Artifacts) == 0 { - return errCatalog("evidence requires at least one hashed artifact") + return errCatalog("evidence requires at least one artifact") } for _, artifact := range evidence.Artifacts { - if !hashPattern.MatchString(artifact.SHA256) { - return fmt.Errorf("artifact %q has an invalid SHA-256 hash", artifact.Path) - } if err := validateRepoPath(artifact.Path); err != nil { return err } @@ -331,9 +333,6 @@ func validateInput(level EvidenceLevel, input Input) error { return errCatalog("git input requires a full lowercase commit revision") } case "fixture": - if !hashPattern.MatchString(input.SHA256) { - return errCatalog("fixture input requires a SHA-256 hash") - } return validateRepoPath(input.Location) case "container": if input.Ref == "" { @@ -382,118 +381,65 @@ func validateRepoPath(path string) error { return nil } -// RefreshArtifacts rewrites every recorded hash from the file it names and -// reports how many changed. Golden files are regenerated by their own tooling, -// so the catalog has to be able to catch up without anyone editing hashes by -// hand. -func (c *Catalog) RefreshArtifacts(root string) (int, error) { - changed := 0 - for evidenceIndex := range c.Evidence { - evidence := &c.Evidence[evidenceIndex] - for artifactIndex := range evidence.Artifacts { - artifact := &evidence.Artifacts[artifactIndex] - actual, err := hashRepositoryFile(root, artifact.Path) - if err != nil { - return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) - } - if actual != artifact.SHA256 { - artifact.SHA256 = actual - changed++ - } - } - for inputIndex := range evidence.Inputs { - input := &evidence.Inputs[inputIndex] - if input.Kind != "fixture" { - continue - } - actual, err := hashRepositoryFile(root, input.Location) - if err != nil { - return changed, fmt.Errorf("evidence %q: %w", evidence.ID, err) - } - if actual != input.SHA256 { - input.SHA256 = actual - changed++ - } - } - } - return changed, nil -} - -// Encode renders the catalog as indented JSON with a trailing newline. -func (c Catalog) Encode() ([]byte, error) { - data, err := json.MarshalIndent(c, "", " ") - if err != nil { - return nil, fmt.Errorf("encode assurance catalog: %w", err) - } - return append(data, '\n'), nil -} - -// VerifyArtifacts confirms every catalog artifact and fixture input still -// matches its recorded hash inside root. -func (c Catalog) VerifyArtifacts(root string) error { +// VerifyFiles confirms every file the catalog names -- each claim's artifacts +// and fixture inputs -- exists inside root as a regular file, and returns how +// many it checked. A claim that points at a file that was renamed or deleted +// would otherwise keep reading as proven. +// +// The count is returned so a caller can refuse a run that reached nothing +// (ADR-0044): a catalog with no files to check and a catalog whose files are +// all present look the same from a nil error. +func (c Catalog) VerifyFiles(root string) (int, error) { + checked := 0 for _, evidence := range c.Evidence { for _, artifact := range evidence.Artifacts { - if err := verifyHashedFile(root, artifact.Path, artifact.SHA256); err != nil { - return fmt.Errorf("evidence %q: %w", evidence.ID, err) + if err := requireRepositoryFile(root, artifact.Path); err != nil { + return checked, fmt.Errorf("evidence %q: %w", evidence.ID, err) } + checked++ } for _, input := range evidence.Inputs { if input.Kind != "fixture" { continue } - if err := verifyHashedFile(root, input.Location, input.SHA256); err != nil { - return fmt.Errorf("evidence %q: %w", evidence.ID, err) + if err := requireRepositoryFile(root, input.Location); err != nil { + return checked, fmt.Errorf("evidence %q: %w", evidence.ID, err) } + checked++ } } - return nil + return checked, nil } -func verifyHashedFile(root, path, want string) error { - actual, err := hashRepositoryFile(root, path) - if err != nil { - return err - } - if actual != want { - return fmt.Errorf("%q hash is %s, want %s", path, actual, want) - } - return nil -} - -// hashRepositoryFile reads one repository file and returns its SHA-256, after -// confirming the path stays inside root. -func hashRepositoryFile(root, path string) (string, error) { +// requireRepositoryFile confirms path names a regular file that stays inside +// root once symlinks are resolved. +func requireRepositoryFile(root, path string) error { if err := validateRepoPath(path); err != nil { - return "", err + return err } resolvedRoot, err := filepath.EvalSymlinks(root) if err != nil { - return "", fmt.Errorf("resolve repository root: %w", err) + return fmt.Errorf("resolve repository root: %w", err) } resolved, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(path))) if err != nil { - return "", fmt.Errorf("resolve %q: %w", path, err) + return fmt.Errorf("resolve %q: %w", path, err) } relative, err := filepath.Rel(resolvedRoot, resolved) if err != nil { - return "", fmt.Errorf("resolve %q relative to repository: %w", path, err) + return fmt.Errorf("resolve %q relative to repository: %w", path, err) } if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("path %q resolves outside the repository", path) + return fmt.Errorf("path %q resolves outside the repository", path) } info, err := os.Stat(resolved) if err != nil { - return "", fmt.Errorf("inspect %q: %w", path, err) + return fmt.Errorf("inspect %q: %w", path, err) } if !info.Mode().IsRegular() { - return "", fmt.Errorf("%q is not a regular file", path) + return fmt.Errorf("%q is not a regular file", path) } - data, err := os.ReadFile(resolved) - if err != nil { - return "", fmt.Errorf("read %q: %w", path, err) - } - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]), nil + return nil } // ChecksForStage returns the catalog checks that belong to one stage. diff --git a/internal/assurance/catalog_test.go b/internal/assurance/catalog_test.go index d2210ce5..cc2fac70 100644 --- a/internal/assurance/catalog_test.go +++ b/internal/assurance/catalog_test.go @@ -23,8 +23,15 @@ func repositoryCatalog(t *testing.T) Catalog { func TestRepositoryCatalogIsValid(t *testing.T) { catalog := repositoryCatalog(t) - if err := catalog.VerifyArtifacts(repositoryRoot); err != nil { - t.Fatalf("catalog artifacts drifted: %v", err) + // Every file a claim names must exist. The count is asserted too: a + // catalog that stopped naming files would otherwise pass this by checking + // nothing (ADR-0044). + checked, err := catalog.VerifyFiles(repositoryRoot) + if err != nil { + t.Fatalf("a claim names a file that is not there: %v", err) + } + if checked < len(catalog.Evidence) { + t.Fatalf("checked %d files for %d evidence claims; every claim names at least one", checked, len(catalog.Evidence)) } for _, stage := range Stages() { if len(catalog.ChecksForStage(stage)) == 0 { @@ -113,7 +120,7 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { }}, Reproduce: [][]string{{"make", "smoke"}}, Artifacts: []EvidenceArtifact{{ - Path: "go.mod", SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + Path: "go.mod", }}, Proves: []string{"It resolves."}, Limitations: []string{"One toolchain."}, @@ -145,13 +152,9 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { c.Evidence[0].Inputs = []Input{{Kind: "workflow", Location: ".github/workflows/smoke.yml"}} }, "evidence without an artifact": func(c *Catalog) { c.Evidence[0].Artifacts = nil }, - "fixture without hash": func(c *Catalog) { - c.Evidence[0].Inputs = []Input{{Kind: "fixture", Location: "go.mod"}} - }, "escaping fixture path": func(c *Catalog) { c.Evidence[0].Inputs = []Input{{ Kind: "fixture", Location: "../../../etc/passwd", - SHA256: "0000000000000000000000000000000000000000000000000000000000000000", }} }, "unsorted checks": func(c *Catalog) { @@ -170,19 +173,50 @@ func TestCatalogRejectsInvalidDocuments(t *testing.T) { } } -func TestVerifyArtifactsDetectsDrift(t *testing.T) { +func TestVerifyFilesRequiresEveryNamedFile(t *testing.T) { root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { t.Fatalf("write fixture: %v", err) } - catalog := Catalog{Evidence: []Evidence{{ - ID: "example", - Artifacts: []EvidenceArtifact{{ - Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", - }}, - }}} - if err := catalog.VerifyArtifacts(root); err == nil { - t.Fatal("expected a hash mismatch to be reported") + if err := os.Mkdir(filepath.Join(root, "directory"), 0o755); err != nil { + t.Fatalf("create directory: %v", err) + } + claim := func(artifact string, inputs ...Input) Catalog { + return Catalog{Evidence: []Evidence{{ + ID: "example", Artifacts: []EvidenceArtifact{{Path: artifact}}, Inputs: inputs, + }}} + } + + present := claim("golden.json", Input{Kind: "fixture", Location: "golden.json"}, + Input{Kind: "git", Location: "https://example.test/repo"}) + checked, err := present.VerifyFiles(root) + if err != nil { + t.Fatalf("files that exist were rejected: %v", err) + } + // The git input is not a repository file and is not counted. + if checked != 2 { + t.Fatalf("checked %d files, want 2", checked) + } + + // Changing a file's contents is not drift any more: the claim names the + // file, and git records which bytes a release had. + if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{\"changed\":true}\n"), 0o644); err != nil { + t.Fatalf("rewrite fixture: %v", err) + } + if _, err := present.VerifyFiles(root); err != nil { + t.Fatalf("a regenerated file was rejected: %v", err) + } + + for name, catalog := range map[string]Catalog{ + "renamed or deleted artifact": claim("gone.json"), + "artifact is a directory": claim("directory"), + "missing fixture input": claim("golden.json", Input{Kind: "fixture", Location: "gone.json"}), + } { + t.Run(name, func(t *testing.T) { + if _, err := catalog.VerifyFiles(root); err == nil { + t.Fatal("expected the missing file to be reported") + } + }) } } @@ -259,34 +293,3 @@ func TestRepositoryCatalogProducesACompleteReport(t *testing.T) { t.Fatal("the coverage matrix is empty") } } - -func TestRefreshArtifactsRewritesDriftedHashes(t *testing.T) { - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "golden.json"), []byte("{}\n"), 0o644); err != nil { - t.Fatalf("write fixture: %v", err) - } - catalog := Catalog{Evidence: []Evidence{{ - ID: "example", - Artifacts: []EvidenceArtifact{{ - Path: "golden.json", SHA256: "1111111111111111111111111111111111111111111111111111111111111111", - }}, - Inputs: []Input{{ - Kind: "fixture", Location: "golden.json", - SHA256: "2222222222222222222222222222222222222222222222222222222222222222", - }}, - }}} - changed, err := catalog.RefreshArtifacts(root) - if err != nil { - t.Fatalf("refresh: %v", err) - } - if changed != 2 { - t.Fatalf("refreshed %d hashes, want 2", changed) - } - if err := catalog.VerifyArtifacts(root); err != nil { - t.Fatalf("refreshed catalog still fails verification: %v", err) - } - again, err := catalog.RefreshArtifacts(root) - if err != nil || again != 0 { - t.Fatalf("second refresh changed %d hashes (err=%v), want 0", again, err) - } -} diff --git a/internal/assurance/cmd/report.go b/internal/assurance/cmd/report.go index e5a857c6..3607b1ee 100644 --- a/internal/assurance/cmd/report.go +++ b/internal/assurance/cmd/report.go @@ -237,8 +237,7 @@ func runCatalogValidate(args []string) error { catalogPath := flags.String("catalog", "", "assurance catalog path") checkID := flags.String("check", "", "print one check") evidenceID := flags.String("evidence", "", "print one evidence claim") - skipArtifacts := flags.Bool("skip-artifacts", false, "skip repository artifact hash verification") - refresh := flags.Bool("refresh", false, "rewrite recorded checksums from the files they name") + skipArtifacts := flags.Bool("skip-artifacts", false, "skip checking that the files the catalog names exist") if err := flags.Parse(args); err != nil { return err } @@ -246,34 +245,19 @@ func runCatalogValidate(args []string) error { if err != nil { return err } - if *refresh { - resolvedRoot, rootErr := repositoryRoot() - if rootErr != nil { - return rootErr - } - changed, refreshErr := catalog.RefreshArtifacts(resolvedRoot) - if refreshErr != nil { - return refreshErr - } - if changed > 0 { - data, encodeErr := catalog.Encode() - if encodeErr != nil { - return encodeErr - } - if err := os.WriteFile(catalogFile, data, 0o644); err != nil { - return fmt.Errorf("write assurance catalog: %w", err) - } - } - fmt.Printf("Refreshed %d recorded checksum(s) in %s.\n", changed, relativeToWorkingDir(catalogFile)) - return nil - } if !*skipArtifacts { resolvedRoot, rootErr := repositoryRoot() if rootErr != nil { return rootErr } - if err := catalog.VerifyArtifacts(resolvedRoot); err != nil { - return err + checked, verifyErr := catalog.VerifyFiles(resolvedRoot) + if verifyErr != nil { + return verifyErr + } + // A run that looked at no files proved nothing about them; say so + // rather than report a clean catalog. + if checked == 0 && len(catalog.Evidence) > 0 { + return fmt.Errorf("the catalog has %d evidence claims but names no files to check", len(catalog.Evidence)) } } fmt.Printf("Validated %s: %d areas, %d checks, %d evidence claims.\n", diff --git a/internal/assurance/testdata/catalog.json b/internal/assurance/testdata/catalog.json index 98993f3d..b68cede1 100644 --- a/internal/assurance/testdata/catalog.json +++ b/internal/assurance/testdata/catalog.json @@ -25,10 +25,22 @@ "stage": "post-release", "level": "advisory", "description": "Runs the same scan several times and records timing.", - "source": { "workflow": "assurance-assessment.yml", "job": "perf-samples" }, - "reproduce": [["make", "benchmark-samples"]], - "proves": ["Repeated runs produced identical normalized output."], - "limitations": ["Timings are observations from one machine, not limits."] + "source": { + "workflow": "assurance-assessment.yml", + "job": "perf-samples" + }, + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "proves": [ + "Repeated runs produced identical normalized output." + ], + "limitations": [ + "Timings are observations from one machine, not limits." + ] }, { "id": "release-checksums", @@ -37,10 +49,23 @@ "stage": "pre-release", "level": "gate", "description": "Hashes every release file and compares it with SHA256SUMS.", - "source": { "workflow": "release.yml", "job": "verify-draft" }, - "reproduce": [["sha256sum", "--check", "SHA256SUMS"]], - "proves": ["Every published file hashes to the recorded value."], - "limitations": ["Checksums prove the files were not altered after they were built."] + "source": { + "workflow": "release.yml", + "job": "verify-draft" + }, + "reproduce": [ + [ + "sha256sum", + "--check", + "SHA256SUMS" + ] + ], + "proves": [ + "Every published file hashes to the recorded value." + ], + "limitations": [ + "Checksums prove the files were not altered after they were built." + ] }, { "id": "smoke", @@ -49,14 +74,36 @@ "stage": "prerequisites", "level": "gate", "description": "Runs scan against pinned public repositories.", - "source": { "workflow": "smoke.yml", "job": "smoke" }, + "source": { + "workflow": "smoke.yml", + "job": "smoke" + }, "expected_instances": [ - { "name": "go", "ecosystems": ["Go"] }, - { "name": "node", "ecosystems": ["JavaScript"] } + { + "name": "go", + "ecosystems": [ + "Go" + ] + }, + { + "name": "node", + "ecosystems": [ + "JavaScript" + ] + } + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], + "proves": [ + "Each ecosystem produces the dependency graph recorded in its golden file." ], - "reproduce": [["make", "smoke"]], - "proves": ["Each ecosystem produces the dependency graph recorded in its golden file."], - "limitations": ["Golden files pin one revision of one example project per ecosystem."] + "limitations": [ + "Golden files pin one revision of one example project per ecosystem." + ] } ], "evidence": [ @@ -76,16 +123,27 @@ "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" } ], - "required_tools": ["git", "go"], - "reproduce": [["make", "smoke"]], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "make", + "smoke" + ] + ], "artifacts": [ { - "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", - "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json" } ], - "proves": ["The Go detector resolves a build-tool-backed module graph."], - "limitations": ["The result depends on a compatible Go toolchain."] + "proves": [ + "The Go detector resolves a build-tool-backed module graph." + ], + "limitations": [ + "The result depends on a compatible Go toolchain." + ] } ] } diff --git a/internal/assurance/testdata/golden/all-pass.report.json b/internal/assurance/testdata/golden/all-pass.report.json index 6b07becd..11adcf59 100644 --- a/internal/assurance/testdata/golden/all-pass.report.json +++ b/internal/assurance/testdata/golden/all-pass.report.json @@ -342,8 +342,7 @@ ], "artifacts": [ { - "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", - "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json" } ], "proves": [ diff --git a/internal/assurance/testdata/golden/mixed-failure.report.json b/internal/assurance/testdata/golden/mixed-failure.report.json index 7b7aa4e7..b85398e5 100644 --- a/internal/assurance/testdata/golden/mixed-failure.report.json +++ b/internal/assurance/testdata/golden/mixed-failure.report.json @@ -354,8 +354,7 @@ ], "artifacts": [ { - "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json", - "sha256": "27a48399b6669f7b6a436080e277877dc9ef3c884634ceb3c0cdcab634eb97d0" + "path": "internal/assurance/testdata/fixtures/all-pass/results/smoke.go.json" } ], "proves": [ diff --git a/internal/detectors/sbom/detector_test.go b/internal/detectors/sbom/detector_test.go index 992dfa91..e649b614 100644 --- a/internal/detectors/sbom/detector_test.go +++ b/internal/detectors/sbom/detector_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -167,7 +168,10 @@ func TestDetectorResolveGraph_RejectsSyftJSON(t *testing.T) { if !strings.Contains(err.Error(), "syft convert") { t.Fatalf("ResolveGraph() error %q does not name the conversion", err.Error()) } - if !strings.Contains(err.Error(), path) { + // The error quotes the path, and quoting doubles a Windows path's + // backslashes, so the bare path is not a substring there. Look for the + // path as the message spells it. + if !strings.Contains(err.Error(), strconv.Quote(path)) { t.Fatalf("ResolveGraph() error %q missing the file path", err.Error()) } } diff --git a/internal/detectors/swiftpm/detector.go b/internal/detectors/swiftpm/detector.go index e60730a2..ce0d3169 100644 --- a/internal/detectors/swiftpm/detector.go +++ b/internal/detectors/swiftpm/detector.go @@ -32,11 +32,18 @@ type Detector struct { // evidence detection is built from it, and position attachment walks it. They // had drifted into three different lists, so a project whose only lockfile sat // in the Xcode workspace was recognized for line numbers but not for reading. +// +// The entries are slash-separated on every platform. The list is published as +// the detector's evidence patterns, so it has to read the same in the support +// matrix and in `bomly plugin list` whatever machine produced them; built with +// filepath.Join it came out backslash-separated on Windows. Reading a file +// needs no conversion: every read joins an entry onto a directory with +// filepath.Join, which rewrites the separators for the host. var resolvedCandidates = []string{ "Package.resolved", ".package.resolved", - filepath.Join(".swiftpm", "xcode", "package.xcworkspace", "xcshareddata", "swiftpm", "Package.resolved"), - filepath.Join("project.xcworkspace", "xcshareddata", "swiftpm", "Package.resolved"), + ".swiftpm/xcode/package.xcworkspace/xcshareddata/swiftpm/Package.resolved", + "project.xcworkspace/xcshareddata/swiftpm/Package.resolved", } // evidencePatterns is what makes this detector applicable: any lockfile diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index f20871a7..af55d1e8 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -9,6 +9,54 @@ FUZZTIME="${FUZZTIME:-60s}" # still exits non-zero when any target failed. FUZZ_RESULTS_JSONL="${FUZZ_RESULTS_JSONL:-}" +# FUZZ_JOBS runs that many targets at once (default 1, one after another). +# Each target is its own `go test` process, so they parallelize cleanly; the +# release prerequisites stage uses this to keep thirty targets from taking +# thirty times one target's budget. Every target is attempted in this mode, +# and each one's output is printed whole when it finishes rather than +# interleaved with its neighbours'. +FUZZ_JOBS="${FUZZ_JOBS:-1}" +case "${FUZZ_JOBS}" in + ''|*[!0-9]*|0) echo "FUZZ_JOBS must be a positive integer, got '${FUZZ_JOBS}'" >&2; exit 2 ;; +esac + +# run_target fuzzes one " " pair, records it when a results +# file is set, and returns the target's exit status. +run_target() { + local package="${1%% *}" fuzz="${1#* }" started status=0 + local -a workers=() + # Concurrent fuzzers would each start one worker per CPU and fight over + # them; give each an even share instead. (The array is expanded with the + # ${x[@]+...} form below because bash 3.2, which macOS ships, treats an + # empty array as unset under `set -u`.) + if [ "${FUZZ_JOBS}" -gt 1 ]; then + local cpus + cpus="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)" + workers=("-parallel=$(( cpus / FUZZ_JOBS > 0 ? cpus / FUZZ_JOBS : 1 ))") + fi + echo "==> go test ${package} -run=^$ -fuzz=^${fuzz}$ -fuzztime=${FUZZTIME}" + started="$(date -u +%s)" + go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" ${workers[@]+"${workers[@]}"} || status=$? + if [ -n "${FUZZ_RESULTS_JSONL}" ]; then + # One short line per append, so concurrent targets cannot tear each other. + printf '{"name":"%s %s","exit_code":%s,"duration_s":%s}\n' \ + "${package##*/}" "${fuzz}" "${status}" "$(( $(date -u +%s) - started ))" \ + >> "${FUZZ_RESULTS_JSONL}" + fi + return "${status}" +} + +# Internal entry point for the parallel mode: run one target, print its output +# in one piece, and exit with its status. +if [ "${1:-}" = "--target" ]; then + output="$(mktemp)" + status=0 + run_target "$2" > "${output}" 2>&1 || status=$? + cat "${output}" + rm -f "${output}" + exit "${status}" +fi + # The SDK's own fuzz targets (package URL canonicalization, graph/registry # transport JSON) moved with the sdk package to the bomly-sdk repository and # run there. @@ -50,25 +98,26 @@ if [ -n "${FUZZ_RESULTS_JSONL}" ]; then fi failures=0 + +if [ "${FUZZ_JOBS}" -gt 1 ]; then + # xargs keeps going past a failing target and exits non-zero if any failed. + if ! printf '%s\0' "${targets[@]}" | xargs -0 -n 1 -P "${FUZZ_JOBS}" "$0" --target; then + echo "one or more fuzz targets failed" >&2 + exit 1 + fi + exit 0 +fi + for target in "${targets[@]}"; do - package="${target%% *}" - fuzz="${target#* }" - echo "==> go test ${package} -run=^$ -fuzz=^${fuzz}$ -fuzztime=${FUZZTIME}" - started="$(date -u +%s)" status=0 if [ -n "${FUZZ_RESULTS_JSONL}" ]; then - go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" || status=$? + run_target "${target}" || status=$? else - go test "${package}" -run=^$ -fuzz="^${fuzz}$" -fuzztime="${FUZZTIME}" + run_target "${target}" fi if [ "${status}" -ne 0 ]; then failures=$((failures + 1)) fi - if [ -n "${FUZZ_RESULTS_JSONL}" ]; then - printf '{"name":"%s %s","exit_code":%s,"duration_s":%s}\n' \ - "${package##*/}" "${fuzz}" "${status}" "$(( $(date -u +%s) - started ))" \ - >> "${FUZZ_RESULTS_JSONL}" - fi done if [ "${failures}" -ne 0 ]; then diff --git a/test/assurance/sdk_contract_test.go b/test/assurance/sdk_contract_test.go index 3c23252f..1a4f20b7 100644 --- a/test/assurance/sdk_contract_test.go +++ b/test/assurance/sdk_contract_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -124,6 +125,13 @@ func TestSDKContractCommandStderrHiddenBelowDebug(t *testing.T) { } } +// unixPermissions reports whether the host has owner/group/other permission +// bits to assert on. Windows does not: access there is governed by ACLs, and +// Go reports every directory as 0777 and every writable file as 0666 whatever +// mode it was created with. The containment half of the cache contract below +// is checked on every platform; only the mode assertions are Unix-only. +const unixPermissions = runtime.GOOS != "windows" + func TestSDKContractFileCachePermissionsAndContainment(t *testing.T) { t.Parallel() @@ -138,7 +146,7 @@ func TestSDKContractFileCachePermissionsAndContainment(t *testing.T) { if err != nil { t.Fatal(err) } - if perm := info.Mode().Perm(); perm != 0o700 { + if perm := info.Mode().Perm(); unixPermissions && perm != 0o700 { t.Fatalf("cache dir permissions = %o, want 700", perm) } @@ -167,7 +175,7 @@ func TestSDKContractFileCachePermissionsAndContainment(t *testing.T) { if err != nil { t.Fatal(err) } - if perm := entryInfo.Mode().Perm(); perm != 0o600 { + if perm := entryInfo.Mode().Perm(); unixPermissions && perm != 0o600 { t.Fatalf("cache entry permissions = %o, want 600", perm) }