diff --git a/.github/workflows/assurance-assessment.yml b/.github/workflows/assurance-assessment.yml index c908acfe..349f723a 100644 --- a/.github/workflows/assurance-assessment.yml +++ b/.github/workflows/assurance-assessment.yml @@ -316,15 +316,6 @@ jobs: if-no-files-found: error retention-days: 30 - sbom: - name: SBOM interoperability - needs: resolve - uses: ./.github/workflows/sbom-interoperability.yml - with: - ref: ${{ needs.resolve.outputs.tag }} - release_tag: ${{ needs.resolve.outputs.tag }} - assurance: true - perf-samples: name: Repeated scan timing needs: resolve @@ -400,7 +391,7 @@ jobs: # always(), so a failed check still gets a published report — but only when # the release itself was resolved, since everything below needs its tag. if: always() && needs.resolve.result == 'success' - needs: [resolve, install-scripts, public-download, released-scan, sbom, perf-samples] + needs: [resolve, install-scripts, public-download, released-scan, perf-samples] runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/.github/workflows/assurance-prerequisites.yml b/.github/workflows/assurance-prerequisites.yml index a24c5b80..e8701345 100644 --- a/.github/workflows/assurance-prerequisites.yml +++ b/.github/workflows/assurance-prerequisites.yml @@ -54,6 +54,18 @@ jobs: assurance: true assurance_tag: ${{ inputs.assurance_tag }} + # The documents Bomly writes are checked by the official SPDX and CycloneDX + # validators here, before a tag exists. This ran after publication until an + # invalid merged SPDX export shipped in v0.28.0: nothing about the check needs + # a published release, and a failure found here costs a pull request. + sbom: + name: SBOM interoperability + uses: ./.github/workflows/sbom-interoperability.yml + with: + ref: ${{ inputs.ref || github.ref }} + assurance: true + assurance_tag: ${{ inputs.assurance_tag }} + # Fuzzing is advisory: a finding is recorded in the report and does not stop a # release. `continue-on-error` is not allowed on a job that calls a reusable # workflow, so fuzz.yml itself skips its failing step when it is called in @@ -132,7 +144,7 @@ jobs: verdict: name: Prerequisites verdict if: always() - needs: [smoke, portable, fuzz, catalog] + needs: [smoke, portable, sbom, fuzz, catalog] runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/sbom-interoperability.yml b/.github/workflows/sbom-interoperability.yml index 48fc74d6..ebf8d617 100644 --- a/.github/workflows/sbom-interoperability.yml +++ b/.github/workflows/sbom-interoperability.yml @@ -2,8 +2,15 @@ # from two source documents — and validates all four with the official SPDX # and CycloneDX tools, pinned by checksum. The merged pair is also checked for # the ADR-0042 rule that a merge links its sources instead of adopting one's -# identity. Runs weekly on its own, and as part of the post-release assessment -# against the binary the release actually shipped. +# identity. +# +# It runs where a failure can still be fixed by an ordinary change: on pull +# requests that touch what decides SBOM output, and in the release +# prerequisites stage before a version is tagged. It used to run after +# publication and on a weekly schedule. The first caught an invalid merged +# SPDX export only once the release was public, and the second had been failing +# on it for three weeks without anyone being told -- a failed scheduled run +# notifies no one. # # The download, generation, and validation logic lives in # internal/assurance/sbominterop, which records every command, exit code, and @@ -12,9 +19,21 @@ name: SBOM interoperability assurance on: - schedule: - - cron: "41 5 * * 1" + pull_request: + # What can change the documents Bomly writes: the pinned SDK, which owns + # the SBOM codec, the detector and command code around it, the inputs, and + # this check itself. + paths: + - go.mod + - go.sum + - internal/detectors/sbom/** + - internal/cli/** + - internal/output/** + - internal/assurance/sbominterop/** + - test/smoke/testdata/sboms/** + - .github/workflows/sbom-interoperability.yml workflow_dispatch: + # Called by "Release prerequisites" (assurance-prerequisites.yml). workflow_call: inputs: ref: @@ -22,16 +41,24 @@ on: type: string required: false default: "" - release_tag: - description: Validate the binary published for this release tag instead of a freshly built one. - type: string - required: false - default: "" assurance: description: Upload a release assurance check result. type: boolean required: false default: false + assurance_tag: + description: Release tag recorded in the check result. + type: string + required: false + default: "" + +# A newer push to a pull request supersedes the run in progress. Nothing else +# is cancelled: when this is called by the release prerequisites stage, +# `github.workflow` is the caller's name, so a manual run cannot share a group +# with it and knock a gate out of a release. +concurrency: + group: sbom-interoperability-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -68,29 +95,8 @@ jobs: java-version: "21" - name: Build the Bomly binary - if: inputs.release_tag == '' run: make build-full - # The post-release assessment validates what users downloaded, not a - # rebuild of it, so the released archive is verified against the published - # checksum list before it is used. - - name: Download the released binary - if: inputs.release_tag != '' - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ inputs.release_tag }} - run: | - set -euo pipefail - version="${RELEASE_TAG#v}" - archive="bomly_${version}_linux_amd64.tar.gz" - mkdir -p "${RUNNER_TEMP}/release" bin - gh release download "${RELEASE_TAG}" --repo bomly-dev/bomly-cli \ - --pattern "${archive}" --pattern SHA256SUMS --dir "${RUNNER_TEMP}/release" - (cd "${RUNNER_TEMP}/release" && sha256sum --check --ignore-missing SHA256SUMS) - tar -xzf "${RUNNER_TEMP}/release/${archive}" -C "${RUNNER_TEMP}/release" bomly - install -m 0755 "${RUNNER_TEMP}/release/bomly" bin/bomly - ./bin/bomly version - - name: Generate and validate SBOMs id: sbom continue-on-error: true @@ -103,7 +109,7 @@ jobs: - name: Record the interoperability result if: always() env: - BOMLY_ASSURANCE_TAG: ${{ inputs.release_tag }} + BOMLY_ASSURANCE_TAG: ${{ inputs.assurance_tag }} run: | set -euo pipefail manifest="${OUTPUT_DIR}/run-manifest.json" diff --git a/AGENTS.md b/AGENTS.md index 56f8d30a..6945a0f5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -361,13 +361,14 @@ Smoke tests (`test/smoke/`, `make smoke`) drive the built binary end-to-end agai Every quality check belongs to one of three release stages and is declared in `docs/assurance/catalog.json` (schema `bomly.assurance-catalog/v1`): -- **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, fuzz, catalog validation). +- **prerequisites** — run on the source tree before a tag exists (smoke, portable stability, cross-builds, SBOM interoperability, fuzz, catalog validation). - **pre-release** — run inside `release.yml` against the still-draft release (asset completeness, checksums, the Sigstore/cosign signature over the checksum list, SLSA build provenance, released binaries). -- **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, SBOM interoperability, performance samples). +- **post-release** — run against the shipped binaries after publication (install scripts, public download, released-binary scans, performance samples). Rules: - Every check writes one `bomly.assurance-check/v1` document per instance through `go run ./internal/assurance/cmd` (`emit`, `gotest`, `convert`, or `verify-release`) and uploads it as an `assurance-*` artifact. Never hand-write that JSON in a workflow. +- Put a check in the earliest stage that has what it needs: one that only needs a built binary belongs in **prerequisites**, where a failure costs a pull request rather than a release. SBOM interoperability ran post-release until v0.28.0 shipped an invalid merged SPDX export it would have stopped. - Adding a check means adding a catalog entry **and** emitting its result; a declared check with no result is reported as `missing` and blocks its stage when it is a gate. - `proves` and `limitations` are mandatory, public, and written in plain language — they are rendered on bomly.dev/assurance. - Public evidence claims live in the same catalog (`evidence[]`), pin what this repository's history cannot address (a Git input by revision, a container by digest), name their fixture and expected-result files by path, and name the check that backs them. `make assurance-catalog` confirms every file they name exists. Do not add a checksum of an in-repository file to the catalog: it is a second copy of what git knows, and it can only drift. diff --git a/dev-docs/CI.md b/dev-docs/CI.md index 32f1b706..66568606 100644 --- a/dev-docs/CI.md +++ b/dev-docs/CI.md @@ -16,7 +16,7 @@ its own minimal CI in its own repository. | `Portable stability assurance` | Manual dispatch, `workflow_call` | Repeated unit tests on Linux, macOS, and Windows plus cross-builds of every release binary | | `Fuzz` | Nightly schedule, dispatch, `workflow_call` | Native Go fuzzing over the `scripts/run-fuzz.sh` target list; uploads minimized failures as artifacts | | `CodeQL` | Pull requests, pushes, schedule | Static analysis for Go and JavaScript | -| `SBOM interoperability assurance` | Weekly schedule, dispatch, `workflow_call` | Validates generated SPDX and CycloneDX documents with checksum-pinned official tools | +| `SBOM interoperability assurance` | Pull requests touching SBOM output, dispatch, `workflow_call` | Validates generated SPDX and CycloneDX documents with checksum-pinned official tools | | `Auto Version` | Manual dispatch | Runs `Release prerequisites`, then bumps the version, tags, and starts `Release` | | `Release` | Release tags | GoReleaser build/publish with signed checksums and SLSA provenance, plus stage 2 of release assurance | | `Release assessment` | `release: published`, dispatch | Stage 3 of release assurance against the published binaries; writes the per-release report | diff --git a/dev-docs/RELEASE_ASSURANCE.md b/dev-docs/RELEASE_ASSURANCE.md index 245f6cb6..4dd8f5dd 100644 --- a/dev-docs/RELEASE_ASSURANCE.md +++ b/dev-docs/RELEASE_ASSURANCE.md @@ -18,13 +18,29 @@ checks are named in each catalog entry's `source`. | Stage | Runs | Workflow | | --- | --- | --- | -| `prerequisites` | Before a tag exists, on the source tree | `assurance-prerequisites.yml`, which calls `smoke.yml`, `portable-assurance.yml`, and `fuzz.yml` | +| `prerequisites` | Before a tag exists, on the source tree | `assurance-prerequisites.yml`, which calls `smoke.yml`, `portable-assurance.yml`, `sbom-interoperability.yml`, and `fuzz.yml` | | `pre-release` | Inside the release pipeline, against the still-draft release | `release.yml` | -| `post-release` | After publication, against the shipped binaries | `assurance-assessment.yml` and `sbom-interoperability.yml` | +| `post-release` | After publication, against the shipped binaries | `assurance-assessment.yml` | A stage passes when every `gate` check in it passes and no declared check is missing. `advisory` checks are always reported and never block. +### Which stage a check belongs in + +Put a check in the earliest stage that has what it needs. A check that only +needs a built binary belongs in `prerequisites`, where a failure costs a pull +request; `pre-release` is for what needs the draft's files; `post-release` is +for what cannot exist before publication — the public download, the install +scripts. + +The SBOM interoperability check is the example of getting this wrong. It ran +after publication, on the reasoning that it should test the binary users +download. It needs only a build. v0.28.0 shipped a merged SPDX export the +official validator rejects, and the check reported it once the release was +public; the same job had also been failing on a weekly schedule for three +weeks, which notifies no one. It now runs in `prerequisites` and on pull +requests that touch what decides SBOM output. + ## The check-result contract Every check writes one JSON file per instance, named `[.].json`, diff --git a/dev-docs/RELEASE_CHECKLIST.md b/dev-docs/RELEASE_CHECKLIST.md index 8901cc8d..084fdf97 100644 --- a/dev-docs/RELEASE_CHECKLIST.md +++ b/dev-docs/RELEASE_CHECKLIST.md @@ -6,7 +6,7 @@ Use this checklist when publishing a tagged Bomly CLI release. - Confirm `main` is green for required checks. - Confirm release publishing credentials are configured in GitHub Actions. -- `Auto Version` runs the `Release prerequisites` stage (smoke, platform stability, cross-builds, fuzz, catalog) on the commit it is about to tag and refuses to tag when it fails, so there is no separate smoke run to start by hand. To pre-flight without tagging: `gh workflow run assurance-prerequisites.yml -f ref=main`. +- `Auto Version` runs the `Release prerequisites` stage (smoke, platform stability, cross-builds, SBOM interoperability, fuzz, catalog) on the commit it is about to tag and refuses to tag when it fails, so there is no separate smoke run to start by hand. To pre-flight without tagging: `gh workflow run assurance-prerequisites.yml -f ref=main`. - If the stage fails, fix the cause on `main` — for stale golden files, run `Update Smoke Goldens` and merge its PR — then start `Auto Version` again. No tag and no release exist yet. ## Release workflow @@ -25,7 +25,7 @@ Use this checklist when publishing a tagged Bomly CLI release. ## After publishing -- `Release assessment` starts automatically once the release is published: it runs the install scripts on all three operating systems, re-downloads the public files, scans real projects with the released binary, validates its SBOM output with the official tools, and records repeated-scan timings. +- `Release assessment` starts automatically once the release is published: it runs the install scripts on all three operating systems, re-downloads the public files, scans real projects with the released binary (including an SBOM as input), and records repeated-scan timings. The official SPDX and CycloneDX validators do not run here: they check the SBOMs Bomly writes in the `Release prerequisites` stage, before the tag exists. - Read the report it publishes at [bomly.dev/assurance](https://bomly.dev/assurance), or the JSON it commits to `docs/assurance/reports/.json`. - If it opens a `Release assurance: ` issue, triage it: the release is already live, so the fix is a follow-up release, not an edit to this one. diff --git a/docs/ASSURANCE.md b/docs/ASSURANCE.md index 7628f7d5..bf2fd06b 100644 --- a/docs/ASSURANCE.md +++ b/docs/ASSURANCE.md @@ -52,8 +52,9 @@ Highlights: provenance are verified against the release files before publication. - **Installation.** The published install scripts are run on all three operating systems against the new release. -- **SBOM interoperability.** The SBOM documents Bomly writes are validated with - the official SPDX and CycloneDX tools, pinned by checksum. +- **SBOM interoperability.** The SBOM documents Bomly writes, including one + merged from two source documents, are validated with the official SPDX and + CycloneDX tools, pinned by checksum, before a version is tagged. - **Speed and stability.** The same scan is repeated with a cold and a warm cache to record timing and confirm the output does not change. diff --git a/docs/assurance/catalog.json b/docs/assurance/catalog.json index 455d7857..35890b26 100644 --- a/docs/assurance/catalog.json +++ b/docs/assurance/catalog.json @@ -468,9 +468,9 @@ "id": "sbom-interoperability", "title": "Official validators accept our SBOMs", "area": "sbom", - "stage": "post-release", + "stage": "prerequisites", "level": "gate", - "description": "Generates SPDX 2.3 and CycloneDX 1.7 documents with the released binary — one pair from a single input and one pair merged from two source documents — and validates all four with checksum-pinned official SPDX and CycloneDX tools.", + "description": "Generates SPDX 2.3 and CycloneDX 1.7 documents from the code about to be released — one pair from a single input and one pair merged from two source documents — and validates all four with checksum-pinned official SPDX and CycloneDX tools.", "source": { "workflow": "sbom-interoperability.yml", "job": "validate" @@ -490,7 +490,8 @@ ], "limitations": [ "Validators check that a document is well formed against its specification. They do not check that every receiving tool interprets it the same way.", - "The merge case uses two checked-in fixtures; it does not cover every combination of source formats." + "The merge case uses two checked-in fixtures; it does not cover every combination of source formats.", + "The documents are generated by a binary built from the release commit, before it is tagged, not by the published archive. The published files are verified separately." ] }, { diff --git a/go.mod b/go.mod index 9f2342d9..fabe7361 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/bomly-dev/bomly-plugin-pyreach-analyzer v0.4.0 github.com/bomly-dev/bomly-plugin-scorecard-matcher v0.3.0 github.com/bomly-dev/bomly-plugin-syft-detector v0.6.0 - github.com/bomly-dev/bomly-sdk v0.14.6 + github.com/bomly-dev/bomly-sdk v0.14.7 github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 diff --git a/go.sum b/go.sum index 9521cdc0..c34674fb 100644 --- a/go.sum +++ b/go.sum @@ -257,8 +257,8 @@ github.com/bomly-dev/bomly-plugin-scorecard-matcher v0.3.0 h1:CB4M5/N7vR2OlMuxKs github.com/bomly-dev/bomly-plugin-scorecard-matcher v0.3.0/go.mod h1:g62nlreSY249tHBl8uoECIan2FW8r+S1xNFZkbdqSBw= github.com/bomly-dev/bomly-plugin-syft-detector v0.6.0 h1:4U/YOzWujQx1LK+VpPhPEpt2ikwiBciDzaE0V5iNN6M= github.com/bomly-dev/bomly-plugin-syft-detector v0.6.0/go.mod h1:B17C01ssOWdoUGa3geCaouPSge+23QiD2ssFLao0g94= -github.com/bomly-dev/bomly-sdk v0.14.6 h1:e5dcUp1Y2XssyMh7xUJJgczdfj0QDPihAY3msEPsq2Q= -github.com/bomly-dev/bomly-sdk v0.14.6/go.mod h1:B/km/swnrS1Tblgpv+5hpVdWHJjyaY5BFdF1cUS38C0= +github.com/bomly-dev/bomly-sdk v0.14.7 h1:zkU07G2q6QFmQNsSFJLhcKdXzj0b2IScuHKxkC7YvjU= +github.com/bomly-dev/bomly-sdk v0.14.7/go.mod h1:B/km/swnrS1Tblgpv+5hpVdWHJjyaY5BFdF1cUS38C0= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= diff --git a/test/assurance/SBOM_INTEROPERABILITY.md b/test/assurance/SBOM_INTEROPERABILITY.md index 48201522..96589408 100644 --- a/test/assurance/SBOM_INTEROPERABILITY.md +++ b/test/assurance/SBOM_INTEROPERABILITY.md @@ -13,12 +13,14 @@ rather than adopting one of them (ADR-0042). Only a real two-source run can produce that document, which is why it is asserted here and not in a unit test. -It runs three ways: on a weekly schedule, when someone starts it from GitHub -Actions, and as part of the post-release assessment. In the post-release run it -downloads the binary the release actually shipped, verifies it against the -published checksum list, and validates the SBOMs that binary produces. It is -kept separate from normal tests because it downloads the validators and takes -longer to run. Bomly never downloads or installs these tools during normal CLI +It runs three ways: on pull requests that change what decides SBOM output (the +pinned SDK, the SBOM detector, the command and output code, the fixtures, or +this check), in the release prerequisites stage before a version is tagged, and +when someone starts it from GitHub Actions. In every case it builds Bomly from +the commit being checked. A failure on a pull request or before a tag is fixed +by an ordinary change; that is why it runs there and no longer after +publication or on a schedule. It is kept separate from normal tests because it +downloads the validators and takes longer to run. Bomly never downloads or installs these tools during normal CLI use. The download, generation, and validation steps live in diff --git a/test/smoke/testdata/golden/lite-scan-sbom-cyclonedx.golden.json b/test/smoke/testdata/golden/lite-scan-sbom-cyclonedx.golden.json index 461bcbeb..9722f659 100644 --- a/test/smoke/testdata/golden/lite-scan-sbom-cyclonedx.golden.json +++ b/test/smoke/testdata/golden/lite-scan-sbom-cyclonedx.golden.json @@ -52,8 +52,8 @@ "detector": "sbom-detector", "document": { "checksum": { - "algorithm": "sha256", - "value": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" + "algorithm": "sha1", + "value": "5ee41e46f9eff45f3d51b2dd53e5c1b71952d53f" }, "created": "2025-01-15T10:00:00Z", "format": "cyclonedx-1.6+json", diff --git a/test/smoke/testdata/golden/lite-scan-sbom-spdx.golden.json b/test/smoke/testdata/golden/lite-scan-sbom-spdx.golden.json index 00c88c8a..663c2c12 100644 --- a/test/smoke/testdata/golden/lite-scan-sbom-spdx.golden.json +++ b/test/smoke/testdata/golden/lite-scan-sbom-spdx.golden.json @@ -43,8 +43,8 @@ "detector": "sbom-detector", "document": { "checksum": { - "algorithm": "sha256", - "value": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + "algorithm": "sha1", + "value": "69cda9c353f8724af58abe7079b978cfb2443d8c" }, "created": "2025-01-15T10:00:00Z", "data_license": "CC0-1.0", diff --git a/test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json b/test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json index 461bcbeb..9722f659 100644 --- a/test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json +++ b/test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json @@ -52,8 +52,8 @@ "detector": "sbom-detector", "document": { "checksum": { - "algorithm": "sha256", - "value": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" + "algorithm": "sha1", + "value": "5ee41e46f9eff45f3d51b2dd53e5c1b71952d53f" }, "created": "2025-01-15T10:00:00Z", "format": "cyclonedx-1.6+json", diff --git a/test/smoke/testdata/golden/scan-sbom-spdx.golden.json b/test/smoke/testdata/golden/scan-sbom-spdx.golden.json index 00c88c8a..663c2c12 100644 --- a/test/smoke/testdata/golden/scan-sbom-spdx.golden.json +++ b/test/smoke/testdata/golden/scan-sbom-spdx.golden.json @@ -43,8 +43,8 @@ "detector": "sbom-detector", "document": { "checksum": { - "algorithm": "sha256", - "value": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + "algorithm": "sha1", + "value": "69cda9c353f8724af58abe7079b978cfb2443d8c" }, "created": "2025-01-15T10:00:00Z", "data_license": "CC0-1.0",