From 1b4611c18b86538a5ca6c867eef11e7945844926 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:16:25 +0000 Subject: [PATCH 01/24] Add CNPG SBOM generator workflow --- .github/workflows/bake_targets.yml | 10 + .github/workflows/sbom-generator.yml | 56 ++- .gitignore | 4 + README.md | 8 + docker-bake.hcl | 8 +- examples/trivy-sbom-examples.txt | 75 ++++ renovate.json | 33 ++ sbom-generator/Dockerfile | 28 ++ sbom-generator/README.md | 236 +++++++++++ sbom-generator/compose.py | 565 +++++++++++++++++++++++++ sbom-generator/generator.py | 399 +++++++++++++++++ sbom-generator/hooks.py | 35 ++ sbom-generator/tests/test_compose.py | 205 +++++++++ sbom-generator/tests/test_generator.py | 100 +++++ sbom-generator/tests/test_hooks.py | 78 ++++ 15 files changed, 1824 insertions(+), 16 deletions(-) create mode 100644 examples/trivy-sbom-examples.txt create mode 100644 sbom-generator/Dockerfile create mode 100644 sbom-generator/README.md create mode 100755 sbom-generator/compose.py create mode 100644 sbom-generator/generator.py create mode 100644 sbom-generator/hooks.py create mode 100644 sbom-generator/tests/test_compose.py create mode 100644 sbom-generator/tests/test_generator.py create mode 100644 sbom-generator/tests/test_hooks.py diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 70f728e4..37bb8b5e 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -46,6 +46,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 + - name: Expose builder stage to SBOM generator + env: + EXTENSION: ${{ inputs.extension_name }} + run: | + sed -i '2i ARG BUILDKIT_SBOM_SCAN_STAGE=builder' "$EXTENSION/Dockerfile" + - name: Build and push uses: docker/bake-action@018cb6412ab401ebaa809aa5f85966b74628600f # v7 id: build @@ -54,7 +60,11 @@ jobs: environment: testing registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} + # renovate: datasource=docker depName=ghcr.io/ardentperf/cnpg-sbom-generator + sbom_generator: ghcr.io/ardentperf/cnpg-sbom-generator:latest with: + # Use the checkout so Bake sees the injected Dockerfile declaration. + source: . files: ./docker-bake.hcl,./${{ inputs.extension_name }}/metadata.hcl push: true diff --git a/.github/workflows/sbom-generator.yml b/.github/workflows/sbom-generator.yml index 3723b873..2319fe37 100644 --- a/.github/workflows/sbom-generator.yml +++ b/.github/workflows/sbom-generator.yml @@ -1,34 +1,64 @@ -name: Build SBOM generator (stub) +name: Build and publish SBOM generator on: - workflow_dispatch: + push: + branches: [main] + paths: + - 'sbom-generator/**' + - '.github/workflows/sbom-generator.yml' + pull_request: + paths: + - 'sbom-generator/**' + - '.github/workflows/sbom-generator.yml' permissions: {} +concurrency: + group: sbom-generator-${{ github.ref }} + cancel-in-progress: false + jobs: build: - name: Build SBOM generator stub runs-on: ubuntu-24.04 permissions: contents: read + packages: write steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false + - name: Run tests + run: python3 -m unittest discover -s sbom-generator/tests -p 'test_*.py' + + - name: Set up QEMU + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 + with: + platforms: linux/arm64 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + + - name: Log in to GHCR + if: github.event_name == 'push' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Build generator stub + - name: Build generator env: - IMAGE: cnpg-sbom-generator:stub-${{ github.sha }} + EVENT_NAME: ${{ github.event_name }} run: | + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/cnpg-sbom-generator" + options=() + if [[ "$EVENT_NAME" == push ]]; then options+=(--push); fi docker buildx build \ - --load \ - --tag "$IMAGE" \ - -f- . <<'DOCKERFILE' - FROM scratch - LABEL org.opencontainers.image.title="cnpg-sbom-generator" - LABEL org.opencontainers.image.description="Workflow-dispatch stub build" - DOCKERFILE \ No newline at end of file + --platform linux/amd64,linux/arm64 \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --tag "$image:latest" \ + --tag "$image:sha-${GITHUB_SHA}" \ + "${options[@]}" \ + sbom-generator diff --git a/.gitignore b/.gitignore index 51b8ae69..8c081895 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ *.so *.dylib +# Python test and bytecode caches +__pycache__/ +*.py[cod] + # Mac .DS_Store diff --git a/README.md b/README.md index 20fb46a5..a6d96748 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,14 @@ docker buildx imagetools inspect --raw | jq '.annotations' skopeo inspect docker:// | jq '.Labels' ``` +## SBOMs and authenticity + +Published images carry platform-specific BuildKit SPDX and provenance +attestations. See the [SBOM and authenticity guide](./sbom-generator/README.md) +for digest-pinned Cosign verification, Buildx extraction, platform-specific +Trivy scanning, and the distinction between payload inventory and direct image +scans. + ## Image catalogs To simplify the deployment of PostgreSQL extensions, this project automatically diff --git a/docker-bake.hcl b/docker-bake.hcl index 382b7f55..24b66182 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -16,6 +16,9 @@ variable "revision" { } fullname = ( environment == "testing") ? "${registry}/${metadata.image_name}-testing" : "${registry}/${metadata.image_name}" +variable "sbom_generator" { + default = "" +} now = timestamp() authors = "The CNPG Extensions Contributors" url = "https://github.com/cnpg-extensions/postgres-extensions-containers" @@ -48,10 +51,9 @@ target "default" { output = [ "type=image,oci-mediatypes=true,oci-artifact=true", ] - attest = [ + attest = concat([ "type=provenance,mode=max", - "type=sbom" - ] + ], sbom_generator == "" ? ["type=sbom"] : ["type=sbom,generator=${sbom_generator}"]) annotations = [ "index,manifest:org.opencontainers.image.created=${now}", "index,manifest:org.opencontainers.image.url=${url}", diff --git a/examples/trivy-sbom-examples.txt b/examples/trivy-sbom-examples.txt new file mode 100644 index 00000000..c0c8ed25 --- /dev/null +++ b/examples/trivy-sbom-examples.txt @@ -0,0 +1,75 @@ +# Trivy SBOM example +# +# This report was generated from the SPDX document extracted from the local +# OCI attestation for pgagent bookworm. Long Debian copyright files were split +# at License: sections before ScanCode ran. The image was built locally with +# BuildKit and pushed to a local registry. GitHub OIDC/cosign attestation +# verification is omitted because it requires a hosted GitHub runner. + +$ docker buildx imagetools inspect 127.0.0.1:5000/pgagent-testing:4.2.3-18-bookworm@sha256:ea4735334c2f494b45b4ba50ac44f66bf55e6e4af56599862e6062341e5eb54e --format '{{ json .SBOM.SPDX }}' > /tmp/pgagent-bookworm-amd64.spdx.json +$ docker run --rm -v /tmp:/work aquasec/trivy:0.74.0 sbom --scanners vuln,license --no-progress --skip-version-check /work/pgagent-bookworm-amd64.spdx.json +2026-09-15T04:04:17Z INFO [vulndb] Need to update DB +2026-09-15T04:04:17Z INFO [vulndb] Downloading vulnerability DB... +2026-09-15T04:04:17Z INFO [vulndb] Downloading artifact... repo="mirror.gcr.io/aquasec/trivy-db:2" +2026-09-15T04:04:30Z INFO [vulndb] Artifact successfully downloaded repo="mirror.gcr.io/aquasec/trivy-db:2" +2026-09-15T04:04:30Z INFO [vuln] Vulnerability scanning is enabled +2026-09-15T04:04:30Z INFO [license] License scanning is enabled +2026-09-15T04:04:30Z INFO Detected SBOM format format="spdx-json" +2026-09-15T04:04:30Z INFO Detected OS family="debian" version="12.15" +2026-09-15T04:04:30Z INFO [debian] Detecting vulnerabilities... os_version="12" pkg_num=3 +2026-09-15T04:04:30Z INFO Number of language-specific files num=0 + +Report Summary + +┌───────────────────────────────────────────────────────┬────────┬─────────────────┬──────────┐ +│ Target │ Type │ Vulnerabilities │ Licenses │ +├───────────────────────────────────────────────────────┼────────┼─────────────────┼──────────┤ +│ /work/pgagent-bookworm-amd64.spdx.json (debian 12.15) │ debian │ 1 │ - │ +├───────────────────────────────────────────────────────┼────────┼─────────────────┼──────────┤ +│ OS Packages │ - │ - │ 3 │ +└───────────────────────────────────────────────────────┴────────┴─────────────────┴──────────┘ +Legend: +- '-': Not scanned +- '0': Clean (no security findings detected) + + +/work/pgagent-bookworm-amd64.spdx.json (debian 12.15) +===================================================== +Total: 1 (UNKNOWN: 0, LOW: 0, MEDIUM: 0, HIGH: 1, CRITICAL: 0) + +┌─────────┬───────────────┬──────────┬──────────┬───────────────────┬───────────────┬──────────────────────────────────────────────────────────────┐ +│ Library │ Vulnerability │ Severity │ Status │ Installed Version │ Fixed Version │ Title │ +├─────────┼───────────────┼──────────┼──────────┼───────────────────┼───────────────┼──────────────────────────────────────────────────────────────┤ +│ pgagent │ CVE-2025-0218 │ HIGH │ affected │ 4.2.3-5.pgdg12+1 │ │ When batch jobs are executed by pgAgent, a script is created │ +│ │ │ │ │ │ │ in... │ +│ │ │ │ │ │ │ https://avd.aquasec.com/nvd/cve-2025-0218 │ +└─────────┴───────────────┴──────────┴──────────┴───────────────────┴───────────────┴──────────────────────────────────────────────────────────────┘ + +OS Packages (license) +===================== +Total: 3 (UNKNOWN: 3, LOW: 0, MEDIUM: 0, HIGH: 0, CRITICAL: 0) + +┌───────────────────────────┬──────────────────────────────────────────────────────────────┬────────────────┬──────────┐ +│ Package │ License │ Classification │ Severity │ +├───────────────────────────┼──────────────────────────────────────────────────────────────┼────────────────┼──────────┤ +│ libboost-filesystem1.74.0 │ Apache-2.0 AND LicenseRef-BSD2 AND LicenseRef-BSD3-DEShaw │ unknown │ UNKNOWN │ +│ │ AND LicenseRef-BSD3-Google AND BSL-1.0 AND │ │ │ +│ │ LicenseRef-Caramel AND LicenseRef-CrystalClear AND │ │ │ +│ │ LicenseRef-HP AND Jam AND LicenseRef-Kempf AND MIT AND │ │ │ +│ │ LicenseRef-NIST AND LicenseRef-OldBoost1 AND │ │ │ +│ │ LicenseRef-OldBoost2 AND LicenseRef-OldBoost3 AND │ │ │ +│ │ LicenseRef-Python AND LicenseRef-SGI AND LicenseRef-Spencer │ │ │ +│ │ AND Zlib │ │ │ +├───────────────────────────┤ │ │ │ +│ libboost-thread1.74.0 │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +│ │ │ │ │ +├───────────────────────────┼──────────────────────────────────────────────────────────────┤ │ │ +│ pgagent │ LicenseRef-scancode-unknown-license-reference AND PostgreSQL │ │ │ +└───────────────────────────┴──────────────────────────────────────────────────────────────┴────────────────┴──────────┘ diff --git a/renovate.json b/renovate.json index 4268ef27..3f9380ef 100644 --- a/renovate.json +++ b/renovate.json @@ -75,6 +75,39 @@ "matchStrings": [ "\\/\\/\\s+renovate: datasource=(?[a-z-.]+?) depName=(?[^\\s]+?)(?: (?:packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))\\s+\\/\\/\\s+\\+default=[\\\"']?[^:]+?:(?[^@]+?)(@(?sha256:[0-9a-f]+))?[\"']?\\s" ] + }, + { + "description": "updates the SBOM generator base image", + "customType": "regex", + "managerFilePatterns": [ + "sbom-generator/Dockerfile" + ], + "matchStrings": [ + "#\\s*renovate: datasource=(?[^\\s]+) depName=(?[^\\s]+)(?: packageName=(?[^\\s]+))?(?: versioning=(?[^\\s]+))?\\s+FROM\\s+(?:[a-zA-Z0-9._/-]+:)?(?[^@\\s]+)(?:@(?sha256:[0-9a-f]+))?" + ] + }, + { + "description": "updates the SBOM generator Dockerfile frontend", + "customType": "regex", + "managerFilePatterns": [ + "sbom-generator/Dockerfile" + ], + "matchStrings": [ + "# syntax=docker/dockerfile:(?[^\\s]+)" + ], + "depNameTemplate": "docker/dockerfile", + "datasourceTemplate": "docker", + "versioningTemplate": "docker" + }, + { + "description": "updates the SBOM generator tool versions", + "customType": "regex", + "managerFilePatterns": [ + "sbom-generator/Dockerfile" + ], + "matchStrings": [ + "#\\s*renovate: datasource=(?[^\\s]+) depName=(?[^\\s]+)(?: versioning=(?[^\\s]+))?\\s+ARG [A-Z0-9_]+_VERSION=(?[^\\s]+)" + ] } ], "packageRules": [ diff --git a/sbom-generator/Dockerfile b/sbom-generator/Dockerfile new file mode 100644 index 00000000..ec7ab001 --- /dev/null +++ b/sbom-generator/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1.7 + +# Python 3.14 is the newest version supported by the pinned ScanCode 32.5.0. +# Move to a newer Python as soon as ScanCode supports it; keep this aligned with SCANCODE_VERSION. +# https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.5.0 +# renovate: datasource=docker depName=python packageName=library/python versioning=docker +FROM python:3.14-slim-bookworm + +ARG TARGETARCH +# renovate: datasource=github-releases depName=anchore/syft versioning=semver +ARG SYFT_VERSION=1.51.1 +# renovate: datasource=pypi depName=scancode-toolkit versioning=pep440 +ARG SCANCODE_VERSION=32.5.0 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y ca-certificates curl gcc g++ libxml2-dev libxslt1-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* \ + && curl --fail --silent --show-error --location \ + "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_${TARGETARCH}.tar.gz" \ + | tar -xz -C /usr/local/bin syft \ + && python -m pip install --no-cache-dir "scancode-toolkit==${SCANCODE_VERSION}" \ + && syft version + +WORKDIR /opt/cnpg-sbom-generator +COPY compose.py generator.py hooks.py ./ + +ENV PYTHONUNBUFFERED=1 +ENTRYPOINT ["python3", "/opt/cnpg-sbom-generator/generator.py"] diff --git a/sbom-generator/README.md b/sbom-generator/README.md new file mode 100644 index 00000000..c0ab75ac --- /dev/null +++ b/sbom-generator/README.md @@ -0,0 +1,236 @@ +# Extension image SBOMs + +BuildKit runs the custom generator as a separate container and mounts the +builder-stage and final scratch filesystems read-only. The generator replaces +the default SBOM scanner; Bake and BuildKit still handle building the image +and attaching its attestations. + +```mermaid +flowchart TD + A[1. Docker Bake submits build] --> B[2. BuildKit builds builder stage,
assembles final scratch filesystem,
and runs generator with both mounted read-only] + B --> C[Builder stage filesystem] + B --> D[Final scratch filesystem] + + subgraph G[Custom generator container] + E[3. Syft: packages, files, ownership] + F[4. Inventory and hash shipped files] + H[6. Match shipped files to builder files
Retain their owning packages] + I[5. ScanCode: shipped licenses] + P[7. Compose SPDX with license metadata] + X[8. Optional downstream augment_spdx hook:
add dependency version and license information] + J[9. Wrap SPDX in an in-toto statement] + E --> H + F --> H + H --> P + I --> P + P --> J + P -.-> X + E -. full builder_document .-> X + X -.-> J + end + + C -->|read-only mount| E + D -->|read-only mount| F + D -->|licenses| I + C -. additional evidence .-> X + J -->|output directory| K[10. BuildKit binds statement to image digest
and publishes SBOM attestation] +``` + +Numbers show the processing sequence for one target/platform; arrows show +data flow. Unnumbered filesystem nodes are inputs. Step 8 runs only when a +downstream hook file is present in the builder filesystem. + +The generator decides what is in the SBOM: shipped files and their identified +packages, plus license and distro metadata. It starts from its own Syft scan +of the builder stage, not Bake's default SBOM. BuildKit pulls the generator +image, supplies the mounts, and incorporates its output into the image index +alongside provenance. + +## Downstream extension hook + +The dashed path in the diagram is an optional extension point. Without a +hook file, the generator wraps the composed SPDX directly. With the +hook enabled, augmentation runs after final-payload +filtering and license composition, before in-toto statement wrapping. + +```python +from hooks import HookContext + +def augment_spdx(document: dict, context: HookContext) -> dict: + """Add additional dependency version and license information.""" + return document +``` + +The context exposes `api_version` (currently `1`), `extension_name`, `platform`, +and read-only filesystem mounts at `builder_path` and `final_path`, plus +`context.builder_document`: the complete, unfiltered SPDX JSON document +produced by the builder-stage Syft scan, before final-file matching or package +filtering. This is the existing parsed `builder_document`, not Syft's native +JSON format, and requires no additional scan. Hooks should treat it as +read-only evidence; the `document` argument is the composed final-payload SPDX +that the hook augments. +Final-file identifiers and checksums are available in `document["files"]`. + +A downstream hook could reuse Syft's full package, file, and relationship +data and read additional evidence from the builder stage to add dependency +version and license information for shipped artifacts to the SPDX document. +Running after filtering prevents these additions from being discarded by the existing +file-ownership selection logic. + +The upstream runner trusts the hook's returned document and wraps it +directly in an in-toto statement. Downstream code is responsible for +producing valid SPDX and for evidence that added components belong to shipped +artifacts. BuildKit receives one completed SPDX statement through the +existing output protocol. + +Place a Python file defining `augment_spdx(document, context)` at +`/usr/local/share/cnpg-sbom/augment_spdx.py` in the extension's builder stage. For +example, add this instruction to that stage, using a source path relative to +the build context: + +```dockerfile +COPY sbom/augment_spdx.py /usr/local/share/cnpg-sbom/augment_spdx.py +``` + +The generator loads that file from the read-only builder mount and calls its +function. No hook file means a no-op. The hook must return the complete SPDX +document; it can modify the supplied document or return a replacement. Load +and hook errors fail the build. There is no additional validation of the +hook's returned SPDX. + +Downstream builds use the same generator image. The hook and its evidence +remain in the builder stage and need not be copied into scratch. The hook +executes in the generator's Python environment, so dependencies installed +only in the builder are not automatically available. Standard-library +processing of precomputed JSON needs no additional generator dependencies. + +## Build locally + +Local builds use Docker Bake directly and do not require the SBOM generator: + +```bash +docker buildx bake -f docker-bake.hcl -f h3/metadata.hcl h3-4_2_3-18-trixie +``` + +The `sbom_generator` Bake variable defaults to empty, which uses BuildKit's +native `type=sbom` generator. Set it to a generator image reference to use the +custom generator. Use `--builder NAME` to select a configured builder, or +`--print` to inspect the Bake definition. + +The separate `sbom-generator.yml` workflow builds the generator when anything +under `sbom-generator/` or the publishing workflow changes. Pull requests validate +the build; changes on `main` publish multi-platform `latest` and `sha-` tags. + +Extension CI consumes the image reference in `bake_targets.yml`. Renovate's +existing Docker digest-pinning configuration tracks `latest`, adding its first +digest after publication and proposing digest updates for subsequent builds. +Generator source changes do not rebuild extensions directly; updating the image +reference in `bake_targets.yml` triggers those builds. When adopting this workflow +in another repository, update both the image reference and its Renovate +`depName` to the publishing owner's GHCR namespace. + +Immediately before Bake, CI edits the checked-out extension Dockerfile to insert +`ARG BUILDKIT_SBOM_SCAN_STAGE=builder` as line 2, after the syntax directive +where present. This exposes the builder +stage to the custom generator. The declaration is required by +[BuildKit stage scanning](https://docs.docker.com/build/metadata/attestations/sbom/#scan-stages); +passing only a build argument does not enable it. + +To test custom SBOM generation locally, publish a generator image accessible +to BuildKit, add the same declaration to your extension Dockerfile, and set +`sbom_generator` to the image reference when running Bake. Ordinary local +builds need neither preparation step. + +## Attestation layout + +Published extension images use BuildKit's SBOM generator protocol. The local +generator emits one in-toto Statement with an SPDX predicate for the platform +payload; BuildKit supplies the final image subject, attestation manifest, and +image index. A multi-platform index therefore contains one image and one +combined provenance/SPDX attestation for each platform. + +The examples below use the H3 image produced by this repository. Replace +`INDEX_DIGEST` with the immutable index digest returned by +`docker buildx imagetools inspect`; the placeholder is intentional because a +tag is not a reproducible security reference. + +## Verify the image + +The retained release workflow is `.github/workflows/bake_targets.yml`. A +production image built from `main` is verified with the workflow identity and +GitHub's OIDC issuer: + +```bash +IMAGE='ghcr.io/cloudnative-pg/h3:4.2.3-18-trixie@sha256:INDEX_DIGEST' + +cosign verify "$IMAGE" \ + --certificate-identity-regexp='^https://github.com/cloudnative-pg/postgres-extensions-containers/.github/workflows/bake_targets\.yml@refs/heads/main$' \ + --certificate-oidc-issuer='https://token.actions.githubusercontent.com' +``` + +Substitute the actual GitHub owner in both the image and identity. A branch or +pull-request build has a different workflow identity. Local validation uses a +disposable Cosign key and `COSIGN_TLOG_UPLOAD=false`; it validates signature +storage and digest binding, not the future hosted OIDC identity. + +## Retrieve the platform SBOM + +Use Buildx's standard platform selector and SPDX template. The extension image +reference is the only difference from the base PostgreSQL extraction form: + +```bash +docker buildx imagetools inspect "$IMAGE" \ + --format '{{ json (index .SBOM "linux/amd64").SPDX }}' \ + > extension-amd64.spdx.json + +trivy sbom --scanners vuln,license extension-amd64.spdx.json +``` + +For the other platform, repeat the same command with `linux/arm64` and a +separate output file: + +```bash +docker buildx imagetools inspect "$IMAGE" \ + --format '{{ json (index .SBOM "linux/arm64").SPDX }}' \ + > extension-arm64.spdx.json + +trivy sbom --scanners vuln,license extension-arm64.spdx.json +``` + +Each report covers one platform. Native-only builds should be extracted and +scanned with `linux/amd64` or `linux/arm64` matching the built image; validate +the second architecture only after the final multi-platform phase. + +The composed document describes the shipped extension payload, including +copied system libraries and `/licenses` files. It does not describe the whole +PostgreSQL container or runtime dependencies supplied by the base image. Scan +the base PostgreSQL image and separately mounted extension images separately. + +## Direct image scans + +Debian-based PostgreSQL images retain installed-package metadata, so a direct +remote image scan can identify OS packages: + +```bash +trivy image --image-src remote --platform linux/amd64 --scanners vuln \ + ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie +``` + +Scratch extension payloads generally do not retain that ownership/version +metadata. The extracted composed SPDX is the intended inventory-based path. +`trivy image` scans the image filesystem and does not consume the BuildKit SBOM +just because the index contains an SBOM attestation; `trivy sbom` consumes the +document extracted above. + +Extraction and vulnerability/license scanning are separate from authenticity. +Cosign verifies the signed index digest. Buildx extraction reads attestation +members whose OCI subject points to a platform image, and the in-toto subject +and descriptor digests bind each statement and blob to that graph. Trivy then +scans the extracted SPDX content; it does not verify the signature. + +For the first hosted rollout, publish the generator before running extension +CI, then let Renovate pin the published `latest` manifest digest. GHCR must be +readable by the consuming workflows and Renovate (through package visibility +or configured credentials). The initial reference has no digest until that +publication; subsequent digest updates select the generator used by extension +builds. Local validation does not exercise hosted OIDC/publication jobs. diff --git a/sbom-generator/compose.py b/sbom-generator/compose.py new file mode 100755 index 00000000..0018319e --- /dev/null +++ b/sbom-generator/compose.py @@ -0,0 +1,565 @@ +#!/usr/bin/env python3 +"""Compose a platform-specific SPDX predicate for the BuildKit SBOM protocol. + +The composer deliberately knows nothing about OCI indexes or in-toto +statements. The generator adds the protocol statement around this predicate; +BuildKit supplies the attestation manifest and binds it to the platform image. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections import defaultdict +from copy import deepcopy +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit + + +EXTENSION_PACKAGE_ID = "SPDXRef-Package-extension-payload" +LICENSE_REF = re.compile(r"LicenseRef-[A-Za-z0-9][A-Za-z0-9.-]*") +LICENSE_OPERATOR = re.compile(r"\s+(?:AND|OR|WITH)\s+") + + +def read_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as stream: + return json.load(stream) + + +def builder_predicate(document: dict[str, Any], path: Path) -> dict[str, Any]: + """Return a raw SPDX document from either Syft or an old attestation. + + The old PR61 fixtures are accepted so the ownership algorithm can be + regression-tested without making an in-toto statement part of the + generator's input or output contract. + """ + + if document.get("predicateType") == "https://spdx.dev/Document": + predicate = document.get("predicate") + else: + predicate = document + if not isinstance(predicate, dict) or predicate.get("SPDXID") != "SPDXRef-DOCUMENT": + raise ValueError(f"{path}: builder evidence is not an SPDX document") + for field in ("packages", "files", "relationships"): + if not isinstance(predicate.get(field), list): + raise ValueError(f"{path}: SPDX {field} must be an array") + return predicate + + +def checksum_key(algorithm: str, value: str) -> tuple[str, str]: + return algorithm.lower(), value.lower() + + +def final_files(document: dict[str, Any], path: Path) -> list[dict[str, str]]: + """Return final file names and checksums from a BuildKit attestation.""" + + files: list[dict[str, str]] = [] + for subject in document["subject"]: + name = subject["name"] + if name.startswith("pkg:"): + raise ValueError( + f"{path}: subject {name!r} is an image subject; use a local-export SBOM" + ) + files.append({ + "name": name.lstrip("/"), + "algorithm": "sha256", + "value": subject["digest"]["sha256"], + }) + if not files: + raise ValueError(f"{path}: final image has no file subjects") + return files + + +def final_inventory_files(inventory: dict[str, Any] | list[dict[str, Any]], path: Path) -> list[dict[str, str]]: + """Validate and normalize the generator's direct final-files inventory.""" + + records = inventory.get("files") if isinstance(inventory, dict) else inventory + if not isinstance(records, list) or not records: + raise ValueError(f"{path}: final filesystem has no files") + + files: list[dict[str, str]] = [] + seen: set[tuple[str, str, str]] = set() + for record in records: + if not isinstance(record, dict): + raise ValueError(f"{path}: final inventory entry is not an object") + name = record.get("name", record.get("fileName")) + if not isinstance(name, str) or not name or name.startswith("pkg:"): + raise ValueError(f"{path}: final inventory has an invalid file name") + checksums = record.get("checksums") + if checksums is not None: + if not isinstance(checksums, list) or len(checksums) != 1: + raise ValueError(f"{path}: final inventory entries need one checksum") + checksum = checksums[0] + algorithm = checksum.get("algorithm") + value = checksum.get("checksumValue") + else: + algorithm = record.get("algorithm", "sha256") + value = record.get("value") + if not isinstance(algorithm, str) or not isinstance(value, str) or not value: + raise ValueError(f"{path}: final inventory entry has no checksum") + normalized = { + "name": name.lstrip("/"), + "algorithm": algorithm.lower(), + "value": value.lower(), + } + identity = (normalized["name"], normalized["algorithm"], normalized["value"]) + if identity not in seen: + files.append(normalized) + seen.add(identity) + files.sort(key=lambda record: (record["name"], record["algorithm"], record["value"])) + return files + + +def path_score(candidate: str, final_name: str) -> tuple[int, int]: + """Prefer an exact path, then the longest shared path suffix.""" + + candidate_parts = tuple(part for part in candidate.lstrip("/").split("/") if part) + final_parts = tuple(part for part in final_name.lstrip("/").split("/") if part) + common_suffix = 0 + for candidate_part, final_part in zip(reversed(candidate_parts), reversed(final_parts)): + if candidate_part != final_part: + break + common_suffix += 1 + return common_suffix, int(candidate_parts == final_parts) + + +def file_id(name: str, algorithm: str, value: str) -> str: + identity = f"{name}\0{algorithm.lower()}:{value.lower()}".encode() + return f"SPDXRef-File-final-{hashlib.sha256(identity).hexdigest()[:24]}" + + +def scancode_licenses( + document: dict[str, Any], +) -> tuple[dict[str, set[str]], dict[str, dict[str, str]]]: + """Return ScanCode SPDX expressions and custom license definitions.""" + + licenses_by_file: defaultdict[str, set[str]] = defaultdict(set) + custom_licenses: set[str] = set() + for record in document.get("files", []): + licenses = { + expression + for detection in record.get("license_detections", []) + if (expression := detection.get("license_expression_spdx")) + and expression not in {"NONE", "NOASSERTION"} + } + if not licenses: + continue + path = record["path"].lstrip("/") + licenses_by_file[path].update(licenses) + custom_licenses.update( + license_id for expression in licenses for license_id in LICENSE_REF.findall(expression) + ) + references = { + reference["spdx_license_key"]: { + "extractedText": reference.get("text") or "NOASSERTION", + "licenseId": reference["spdx_license_key"], + "name": reference.get("name") or reference["spdx_license_key"], + } + for reference in document.get("license_references", []) + if reference.get("spdx_license_key") in custom_licenses + } + for license_id in custom_licenses: + references.setdefault(license_id, { + "extractedText": "NOASSERTION", + "licenseId": license_id, + "name": license_id, + }) + return licenses_by_file, references + + +def debian_os_package(packages: list[dict[str, Any]], path: Path) -> dict[str, Any]: + distros = { + distro + for package in packages + for reference in package.get("externalRefs", []) + if reference.get("referenceType") == "purl" + for distro in parse_qs(urlsplit(reference["referenceLocator"]).query).get("distro", []) + } + if len(distros) != 1 or not next(iter(distros), "").startswith("debian-"): + raise ValueError(f"{path}: packages must identify one Debian distro") + version = next(iter(distros)).removeprefix("debian-") + return { + "SPDXID": f"SPDXRef-OperatingSystem-debian-{version}", + "copyrightText": "NOASSERTION", + "downloadLocation": "NONE", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "debian", + "primaryPackagePurpose": "OPERATING-SYSTEM", + "versionInfo": version, + } + + +def set_document_namespace(document: dict[str, Any], extension_name: str, platform: str) -> None: + # SPDX element IDs are scoped by namespace; hash the content to prevent + # distinct SBOMs from sharing identities when consumers combine them. + content = {key: value for key, value in document.items() if key != "documentNamespace"} + digest = hashlib.sha256( + json.dumps(content, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + document["documentNamespace"] = ( + "https://github.com/cnpg-extensions/postgres-extensions-containers/" + f"sbom-generator/v1/documents/{extension_name}/{platform.replace('/', '-')}-{digest}" + ) + + +def compose(builder_document: dict[str, Any], *, + extension_name: str, + builder_path: Path = Path("builder"), + scancode_report: dict[str, Any] | None = None, + final_inventory: dict[str, Any] | list[dict[str, Any]] | None = None, + platform: str | None = None, + evidence: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Return one raw, platform-specific SPDX predicate. + + ``final_inventory`` is the normal path. ``subject`` handling remains as a + compatibility fixture for the original composer tests, but the generator + never needs a final image/index digest before it writes its predicate. + """ + + builder = builder_predicate(builder_document, builder_path) + licenses_by_file, custom_licenses = scancode_licenses(scancode_report or {}) + final = ( + final_inventory_files(final_inventory, builder_path) + if final_inventory is not None + else final_files(builder_document, builder_path) + ) + builder_records = builder["files"] + relationships = builder["relationships"] + packages = builder["packages"] + + builder_packages = { + package["SPDXID"]: package + for package in packages + if package.get("primaryPackagePurpose") != "FILE" + } + + all_package_ids = {package["SPDXID"] for package in packages} + package_ids = set(builder_packages) + package_ids_by_name: defaultdict[str, set[str]] = defaultdict(set) + for package_id, package in builder_packages.items(): + package_ids_by_name[package["name"]].add(package_id) + retained_package_ids: set[str] = set() + owners_by_source_file: defaultdict[str, set[str]] = defaultdict(set) + for relationship in relationships: + if relationship["relationshipType"] != "CONTAINS": + continue + package_id = relationship["spdxElementId"] + source_file_id = relationship["relatedSpdxElement"] + if package_id in package_ids: + owners_by_source_file[source_file_id].add(package_id) + + by_checksum: defaultdict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + all_file_ids = {record["SPDXID"] for record in builder_records} + for record in builder_records: + for checksum in record["checksums"]: + by_checksum[checksum_key( + checksum["algorithm"], checksum["checksumValue"] + )].append(record) + + composed_files: list[dict[str, Any]] = [] + source_to_final: defaultdict[str, set[str]] = defaultdict(set) + direct_final_owners: defaultdict[str, set[str]] = defaultdict(set) + final_ids: set[str] = set() + + def add_synthetic_file(record: dict[str, str], owner: str | None = None) -> None: + output_record = { + "SPDXID": file_id(record["name"], record["algorithm"], record["value"]), + "checksums": [{ + "algorithm": record["algorithm"].upper(), + "checksumValue": record["value"], + }], + "copyrightText": "NOASSERTION", + "fileName": record["name"], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": ["NOASSERTION"], + } + composed_files.append(output_record) + final_ids.add(output_record["SPDXID"]) + if owner is not None: + retained_package_ids.add(owner) + direct_final_owners[output_record["SPDXID"]].add(owner) + + for final_record in final: + final_name = final_record["name"].lstrip("/") + license_parts = final_name.split("/", 2) + if license_parts[0] == "licenses" and len(license_parts) > 1: + owners = package_ids_by_name.get(license_parts[1], set()) + add_synthetic_file(final_record, next(iter(owners)) if len(owners) == 1 else None) + continue + + candidates = by_checksum.get( + checksum_key(final_record["algorithm"], final_record["value"]), [] + ) + if not candidates: + add_synthetic_file(final_record) + continue + + owned_candidates = [ + record for record in candidates if record["SPDXID"] in owners_by_source_file + ] + candidates = owned_candidates or candidates + best_score = max(path_score(record["fileName"], final_name) for record in candidates) + selected = [ + record for record in candidates + if path_score(record["fileName"], final_name) == best_score + ] + selected.sort(key=lambda record: record["SPDXID"]) + source_names = {record["fileName"].lstrip("/") for record in selected} + if len(source_names) > 1: + add_synthetic_file(final_record) + continue + + source = selected[0] + new_id = file_id(final_record["name"], final_record["algorithm"], final_record["value"]) + output_record = source.copy() + output_record["SPDXID"] = new_id + output_record["fileName"] = final_record["name"] + composed_files.append(output_record) + final_ids.add(new_id) + for record in selected: + source_to_final[record["SPDXID"]].add(new_id) + + for record in composed_files: + licenses = licenses_by_file.get(record["fileName"].lstrip("/")) + if not licenses: + continue + record["licenseInfoInFiles"] = sorted( + set(record.get("licenseInfoInFiles", [])) + .union(licenses) + - {"NONE", "NOASSERTION"} + ) + + owned_final_ids: set[str] = set(direct_final_owners) + for source_file_id, package_ids_for_file in owners_by_source_file.items(): + final_ids_for_source = source_to_final.get(source_file_id) + if not final_ids_for_source: + continue + retained_package_ids.update(package_ids_for_file) + owned_final_ids.update(final_ids_for_source) + + extension_file_ids = final_ids - owned_final_ids + + if extension_file_ids: + retained_package_ids.add(EXTENSION_PACKAGE_ID) + + composed_relationships: list[dict[str, Any]] = [] + seen_relationships: set[str] = set() + for relationship in relationships: + element_id = relationship["spdxElementId"] + related_id = relationship["relatedSpdxElement"] + if ( + (element_id in all_file_ids and element_id not in source_to_final) + or (related_id in all_file_ids and related_id not in source_to_final) + ): + continue + if ( + (element_id in all_package_ids and element_id not in retained_package_ids) + or (related_id in all_package_ids and related_id not in retained_package_ids) + ): + continue + if element_id not in source_to_final and related_id not in source_to_final: + composed_relationships.append(relationship.copy()) + continue + + element_ids = source_to_final.get(element_id, {element_id}) + related_ids = source_to_final.get(related_id, {related_id}) + for new_element_id in element_ids: + for new_related_id in related_ids: + replacement = relationship.copy() + replacement["spdxElementId"] = new_element_id + replacement["relatedSpdxElement"] = new_related_id + identity = json.dumps(replacement, sort_keys=True, separators=(",", ":")) + if identity not in seen_relationships: + seen_relationships.add(identity) + composed_relationships.append(replacement) + + composed_relationships.extend( + { + "spdxElementId": package_id, + "relationshipType": "CONTAINS", + "relatedSpdxElement": file_id_value, + } + for file_id_value, package_ids_for_file in direct_final_owners.items() + for package_id in sorted(package_ids_for_file) + ) + + output = deepcopy(builder) + os_package = debian_os_package(packages, builder_path) + output["packages"] = [ + deepcopy(package) for package in packages + if package["SPDXID"] in retained_package_ids + ] + if extension_file_ids: + output["packages"].append({ + "SPDXID": EXTENSION_PACKAGE_ID, + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": True, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": f"{extension_name}-extension-artifacts", + "supplier": "NOASSERTION", + "versionInfo": "NOASSERTION", + }) + output["packages"].append(os_package) + output["files"] = composed_files + output["relationships"] = composed_relationships + if extension_file_ids: + composed_relationships.extend( + { + "spdxElementId": EXTENSION_PACKAGE_ID, + "relationshipType": "CONTAINS", + "relatedSpdxElement": file_id_value, + } + for file_id_value in sorted(extension_file_ids) + ) + package_by_id = {package["SPDXID"]: package for package in output["packages"]} + file_by_id = {record["SPDXID"]: record for record in composed_files} + licenses_by_package: defaultdict[str, set[str]] = defaultdict(set) + for relationship in composed_relationships: + package = package_by_id.get(relationship.get("spdxElementId")) + file = file_by_id.get(relationship.get("relatedSpdxElement")) + if ( + relationship.get("relationshipType") == "CONTAINS" + and package is not None + and file is not None + and package.get("filesAnalyzed", True) is not False + ): + licenses_by_package[package["SPDXID"]].update( + set(file.get("licenseInfoInFiles", [])) - {"NONE", "NOASSERTION"} + ) + for package_id, licenses in licenses_by_package.items(): + if licenses: + package = package_by_id[package_id] + package["licenseInfoFromFiles"] = sorted( + set(package.get("licenseInfoFromFiles", [])) + .union(licenses) + - {"NONE", "NOASSERTION"} + ) + for package in output["packages"]: + if package.get("licenseDeclared") not in {None, "NONE", "NOASSERTION"}: + continue + file_licenses = set(package.get("licenseInfoFromFiles", [])) - {"NONE", "NOASSERTION"} + if not file_licenses: + continue + package["licenseDeclared"] = " AND ".join( + f"({term})" if LICENSE_OPERATOR.search(term) else term + for term in sorted(file_licenses) + ) + extracted = { + item["licenseId"]: item + for item in output.get("hasExtractedLicensingInfos", []) + } + for license_id in custom_licenses: + extracted.setdefault(license_id, custom_licenses[license_id]) + output["hasExtractedLicensingInfos"] = [ + extracted[key] for key in sorted(extracted) + ] + described_ids = { + relationship["relatedSpdxElement"] + for relationship in composed_relationships + if relationship.get("spdxElementId") == output["SPDXID"] + and relationship.get("relationshipType") == "DESCRIBES" + } + for package_id in sorted(retained_package_ids): + if package_id not in described_ids: + output["relationships"].append({ + "spdxElementId": output["SPDXID"], + "relationshipType": "DESCRIBES", + "relatedSpdxElement": package_id, + }) + output["relationships"].append({ + "spdxElementId": output["SPDXID"], + "relationshipType": "DESCRIBES", + "relatedSpdxElement": os_package["SPDXID"], + }) + output["relationships"].extend( + { + "spdxElementId": os_package["SPDXID"], + "relationshipType": "CONTAINS", + "relatedSpdxElement": package["SPDXID"], + } + for package in output["packages"] + if package["SPDXID"] != os_package["SPDXID"] + and any( + reference.get("referenceType") == "purl" + and reference.get("referenceLocator", "").startswith("pkg:deb/debian/") + for reference in package.get("externalRefs", []) + ) + ) + output["name"] = f"{extension_name}-sbom" + if platform is not None: + if platform not in {"linux/amd64", "linux/arm64"}: + raise ValueError(f"unsupported target platform: {platform!r}") + creation_info = output.setdefault("creationInfo", {}) + creators = list(creation_info.get("creators", [])) + generator_creator = "Tool: cnpg-sbom-generator" + if generator_creator not in creators: + creators.append(generator_creator) + creation_info["creators"] = creators + metadata = { + "generator": "cnpg-sbom-generator", + "generatorVersion": "1", + "platform": platform, + "evidence": evidence or {}, + } + output["annotations"] = list(output.get("annotations", [])) + [{ + "annotationDate": creation_info.get("created", "1970-01-01T00:00:00Z"), + "annotationType": "OTHER", + "annotator": generator_creator, + "comment": json.dumps(metadata, sort_keys=True, separators=(",", ":")), + "spdxElementId": "SPDXRef-DOCUMENT", + }] + if platform is not None: + set_document_namespace(output, extension_name, platform) + return output + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Compose one platform-specific final-payload SPDX predicate" + ) + parser.add_argument("--builder-sbom", type=Path, required=True) + parser.add_argument("--final-inventory", type=Path, required=True) + parser.add_argument("--platform", required=True) + parser.add_argument( + "--scancode-report", + type=Path, + help="Optional ScanCode JSON report for shipped license files", + ) + parser.add_argument("--evidence", type=Path, help="Optional reproducibility evidence JSON") + parser.add_argument("--extension-name", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + output = compose( + read_json(args.builder_sbom), + extension_name=args.extension_name, + builder_path=args.builder_sbom, + final_inventory=read_json(args.final_inventory), + platform=args.platform, + scancode_report=read_json(args.scancode_report) if args.scancode_report else {}, + evidence=read_json(args.evidence) if args.evidence else {}, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as stream: + json.dump(output, stream, indent=2) + stream.write("\n") + print( + f"composed {len(output['packages'])} packages, " + f"{len(output['files'])} final files, " + f"{len(output['relationships'])} relationships", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py new file mode 100644 index 00000000..927159f4 --- /dev/null +++ b/sbom-generator/generator.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +"""BuildKit SBOM generator for the final extension payload. + +The scanner consumes only the files BuildKit mounts for this invocation. The +builder stage is evidence for package ownership; the final stage is inventoried +directly. The output is one BuildKit SBOM-bundle entry: an in-toto Statement +whose predicate is the composed SPDX document. BuildKit owns the OCI +attestation and image/index binding around that statement. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import platform as host_platform +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, Sequence + +from compose import compose, set_document_namespace +from hooks import HookContext, run_augmentation_hook + + +PLATFORM_ARCHITECTURES = { + "amd64": "linux/amd64", + "x86_64": "linux/amd64", + "arm64": "linux/arm64", + "aarch64": "linux/arm64", +} + +INTOTO_STATEMENT_TYPE = "https://in-toto.io/Statement/v1" +SPDX_PREDICATE_TYPE = "https://spdx.dev/Document" + + +def require_directory(value: str | None, variable: str) -> Path: + if not value: + raise RuntimeError(f"{variable} is required") + path = Path(value) + if not path.is_dir(): + raise RuntimeError(f"{variable} does not name a directory: {path}") + return path + + +def find_builder(extra_root: Path) -> Path: + """Find the explicitly requested ``builder`` stage in BuildKit extras.""" + + candidates = [path for path in extra_root.iterdir() if path.name == "sbom-builder"] + if len(candidates) != 1 or not candidates[0].is_dir(): + names = ", ".join(sorted(path.name for path in extra_root.iterdir())) + raise RuntimeError( + "BUILDKIT_SCAN_SOURCE_EXTRAS must contain exactly one sbom-builder " + f"mount; found [{names}]" + ) + return candidates[0] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def final_inventory(root: Path) -> dict[str, Any]: + """Inventory regular files and symlinks without leaving ``root``.""" + + records: list[dict[str, Any]] = [] + + def add_symlink(path: Path) -> None: + relative = path.relative_to(root).as_posix() + stat = path.lstat() + records.append({ + "name": relative, + "algorithm": "sha256", + "value": hashlib.sha256(os.readlink(path).encode()).hexdigest(), + "kind": "symlink", + "mode": stat.st_mode & 0o7777, + }) + + for directory, directory_names, file_names in os.walk(root, followlinks=False): + directory_names.sort() + file_names.sort() + for name in list(directory_names): + path = Path(directory) / name + if path.is_symlink(): + # os.walk lists symlinked directories separately. Record the + # link itself but never recurse through a target outside root. + add_symlink(path) + directory_names.remove(name) + for name in file_names: + path = Path(directory) / name + relative = path.relative_to(root).as_posix() + stat = path.lstat() + if path.is_symlink(): + # Hash the link payload, never its target. This keeps links + # outside the mounted filesystem from being followed. + add_symlink(path) + continue + elif path.is_file(): + value = sha256_file(path) + kind = "file" + else: + raise RuntimeError(f"unsupported final filesystem entry: {path}") + records.append({ + "name": relative, + "algorithm": "sha256", + "value": value, + "kind": kind, + "mode": stat.st_mode & 0o7777, + }) + if not records: + raise RuntimeError(f"final filesystem is empty: {root}") + return {"files": records} + + +def run_json_command(command: Sequence[str], output: Path) -> dict[str, Any]: + try: + subprocess.run( + [*command, f"spdx-json={output}"], + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise RuntimeError(f"required scanner is unavailable: {command[0]}") from error + except subprocess.CalledProcessError as error: + detail = (error.stderr or error.stdout or "scanner failed").strip() + raise RuntimeError(f"{' '.join(command)} failed: {detail}") from error + try: + with output.open(encoding="utf-8") as stream: + document = json.load(stream) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"scanner did not produce valid SPDX JSON at {output}") from error + if not isinstance(document, dict): + raise RuntimeError(f"scanner output is not a JSON object: {output}") + return document + + +def scan_builder(builder: Path, temporary: Path) -> dict[str, Any]: + fixture = os.getenv("BUILDKIT_BUILDER_SPDX") + if fixture: + path = Path(fixture) + if not path.is_file(): + raise RuntimeError(f"BUILDKIT_BUILDER_SPDX does not exist: {path}") + with path.open(encoding="utf-8") as stream: + document = json.load(stream) + if not isinstance(document, dict): + raise RuntimeError("BUILDKIT_BUILDER_SPDX is not a JSON object") + return document + + syft = shutil.which("syft") + if not syft: + raise RuntimeError("syft is required to scan the mounted builder stage") + output = temporary / "builder.spdx.json" + return run_json_command( + [syft, f"dir:{builder}", "--scope", "all-layers", "--quiet", "--output"], + output, + ) + + +def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: + licenses = final_root / "licenses" + if not licenses.exists(): + return {"files": []} + scancode = shutil.which("scancode") + if not scancode: + raise RuntimeError("scancode is required when the final payload has /licenses") + scan_root = prepare_license_scan_root(final_root, temporary) + output = temporary / "scancode.json" + try: + subprocess.run( + [ + scancode, + "--license", + "--license-references", + "--json", + str(output), + str(scan_root), + ], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as error: + detail = (error.stderr or error.stdout or "scancode failed").strip() + raise RuntimeError(f"scancode failed: {detail}") from error + try: + with output.open(encoding="utf-8") as stream: + report = json.load(stream) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"scancode did not produce valid JSON at {output}") from error + if not isinstance(report, dict): + raise RuntimeError("scancode output is not a JSON object") + normalize_scancode_report_paths(report, scan_root, final_root) + return report + + +def prepare_license_scan_root(final_root: Path, temporary: Path) -> Path: + """Split shipped copyright files before ScanCode scans them. + + Debian copyright files can contain thousands of ``License:`` sections. + ScanCode's per-file timeout applies to the combined file, so scan a + directory of chunks instead and map the chunk paths back afterward. + """ + + licenses = final_root / "licenses" + scan_root = temporary / "license-chunks" / "licenses" + for license_file in sorted(licenses.rglob("*")): + if license_file.is_symlink() or not license_file.is_file(): + continue + relative = license_file.relative_to(licenses) + chunk_directory = scan_root / relative + chunk_directory.mkdir(parents=True, exist_ok=True) + prefix = chunk_directory / "license-" + try: + result = subprocess.run( + [ + "csplit", + "-s", + "-z", + "-f", + str(prefix), + str(license_file), + "/^License:/", + "{*}", + ], + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise RuntimeError("csplit is required to prepare license files") from error + if result.returncode: + detail = (result.stderr or result.stdout or "csplit failed").strip() + raise RuntimeError(f"csplit failed for {license_file}: {detail}") + return scan_root + + +def normalize_scancode_report_paths( + report: dict[str, Any], scan_root: Path, final_root: Path +) -> None: + """Collapse split-file paths to the original final-image paths.""" + + scan_prefix = str(scan_root).rstrip("/") + final_prefix = str(final_root).rstrip("/") + chunk_suffix = re.compile(r"/license-[0-9]+$") + + def normalize(path: str) -> str: + if path == scan_prefix: + path = scan_root.name + elif path.startswith(f"{scan_prefix}/"): + path = f"{scan_root.name}/{path[len(scan_prefix) + 1:]}" + elif path == final_prefix: + path = final_root.name + elif path.startswith(f"{final_prefix}/"): + path = f"{final_root.name}/{path[len(final_prefix) + 1:]}" + return chunk_suffix.sub("", path.lstrip("/")) + + for record in report.get("files", []): + path = record.get("path") + if isinstance(path, str): + record["path"] = normalize(path) + for detection in record.get("license_detections", []): + for match in detection.get("matches", []): + from_file = match.get("from_file") + if isinstance(from_file, str): + match["from_file"] = normalize(from_file) + + +def infer_platform(builder_document: dict[str, Any]) -> str: + explicit = os.getenv("SBOM_TARGET_PLATFORM") or os.getenv("BUILDKIT_SCAN_PLATFORM") + if explicit: + if explicit not in {"linux/amd64", "linux/arm64"}: + raise RuntimeError(f"unsupported target platform: {explicit}") + return explicit + + architectures: set[str] = set() + for package in builder_document.get("packages", []): + for reference in package.get("externalRefs", []): + if reference.get("referenceType") != "purl": + continue + locator = reference.get("referenceLocator", "") + for part in locator.split("?")[-1].split("&"): + key, _, value = part.partition("=") + if key == "arch" and value in PLATFORM_ARCHITECTURES: + architectures.add(value) + platforms = {PLATFORM_ARCHITECTURES[architecture] for architecture in architectures} + if len(platforms) != 1: + raise RuntimeError( + "cannot determine target platform from builder package evidence; " + "set SBOM_TARGET_PLATFORM explicitly" + ) + return next(iter(platforms)) + + +def tool_version(command: str) -> str: + executable = shutil.which(command) + if not executable: + return "unavailable" + try: + result = subprocess.run( + [executable, "--version"], check=False, capture_output=True, text=True + ) + except OSError: + return "unavailable" + return (result.stdout or result.stderr).splitlines()[0] if (result.stdout or result.stderr) else "unknown" + + +def statement_for(predicate: dict[str, Any]) -> dict[str, Any]: + """Wrap one SPDX predicate in the statement format BuildKit unbundles. + + BuildKit v0.32's bundle exporter parses scanner output as an in-toto + Statement. It replaces the empty subject with the final image subject when + it attaches the statement, so the generator does not guess an image digest. + """ + + return { + "_type": INTOTO_STATEMENT_TYPE, + "predicateType": SPDX_PREDICATE_TYPE, + "predicate": predicate, + "subject": [], + } + + +def generate() -> Path: + source = require_directory(os.getenv("BUILDKIT_SCAN_SOURCE"), "BUILDKIT_SCAN_SOURCE") + extras = require_directory( + os.getenv("BUILDKIT_SCAN_SOURCE_EXTRAS"), "BUILDKIT_SCAN_SOURCE_EXTRAS" + ) + destination = require_directory( + os.getenv("BUILDKIT_SCAN_DESTINATION"), "BUILDKIT_SCAN_DESTINATION" + ) + builder = find_builder(extras) + if any(destination.iterdir()): + raise RuntimeError(f"scanner output directory must be empty: {destination}") + + extension_name = os.getenv("SBOM_EXTENSION_NAME", "extension") + with tempfile.TemporaryDirectory(prefix="cnpg-sbom-") as temporary_name: + temporary = Path(temporary_name) + builder_document = scan_builder(builder, temporary) + platform = infer_platform(builder_document) + inventory = final_inventory(source) + report = scan_licenses(source, temporary) + evidence = { + "builderSha256": sha256_file(Path(os.getenv("BUILDKIT_BUILDER_SPDX"))) + if os.getenv("BUILDKIT_BUILDER_SPDX") + else "generated-by-syft", + "platformEvidence": "builder package purl architecture", + "hostArchitecture": host_platform.machine(), + "tools": { + "syft": tool_version("syft"), + "scancode": tool_version("scancode"), + "python": sys.version.split()[0], + }, + } + predicate = compose( + builder_document, + extension_name=extension_name, + builder_path=builder, + final_inventory=inventory, + platform=platform, + scancode_report=report, + evidence=evidence, + ) + predicate = run_augmentation_hook(predicate, HookContext( + extension_name=extension_name, + platform=platform, + builder_path=builder, + final_path=source, + builder_document=builder_document, + )) + # Include any downstream augmentation in the final document identity. + set_document_namespace(predicate, extension_name, platform) + statement = statement_for(predicate) + output = destination / "final-payload.spdx.json" + with output.open("w", encoding="utf-8") as stream: + json.dump(statement, stream, indent=2, sort_keys=True) + stream.write("\n") + return output + + +def main() -> int: + try: + output = generate() + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(f"sbom-generator: {error}", file=sys.stderr) + return 1 + print(f"wrote {output}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sbom-generator/hooks.py b/sbom-generator/hooks.py new file mode 100644 index 00000000..cf82e18b --- /dev/null +++ b/sbom-generator/hooks.py @@ -0,0 +1,35 @@ +"""Versioned downstream extension API for the composed payload SPDX.""" + +from dataclasses import dataclass +from pathlib import Path +from runpy import run_path +from typing import Any + + +@dataclass(frozen=True) +class HookContext: + """Filesystem paths and unfiltered scan evidence available to a hook. + + Treat builder_document as read-only evidence. Filesystem paths are valid + only during this invocation. The frozen context does not freeze nested JSON. + """ + + extension_name: str + platform: str + builder_path: Path + final_path: Path + builder_document: dict[str, Any] + api_version: int = 1 + + +def run_augmentation_hook(document: dict[str, Any], context: HookContext) -> dict[str, Any]: + """Run the builder's optional hook file, or return the document unchanged.""" + + hook_path = context.builder_path / "usr/local/share/cnpg-sbom/augment_spdx.py" + if not hook_path.exists(): + return document + try: + namespace = run_path(str(hook_path)) + return namespace["augment_spdx"](document, context) + except Exception as error: + raise RuntimeError(f"SBOM hook {hook_path} failed: {error}") from error diff --git a/sbom-generator/tests/test_compose.py b/sbom-generator/tests/test_compose.py new file mode 100644 index 00000000..de0b2830 --- /dev/null +++ b/sbom-generator/tests/test_compose.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +import copy +import hashlib +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1])) + +from compose import compose, final_inventory_files, set_document_namespace # noqa: E402 + + +def checksum(value): + return [{"algorithm": "SHA256", "checksumValue": value}] + + +def package(spdxid, name, version, purl): + return { + "SPDXID": spdxid, + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [{ + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": purl, + "referenceType": "purl", + }], + "filesAnalyzed": True, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": name, + "supplier": "NOASSERTION", + "versionInfo": version, + } + + +def builder_document(): + return { + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "builder", + "creationInfo": {"created": "2026-01-01T00:00:00Z", "creators": ["Tool: syft"]}, + "packages": [ + package("SPDXRef-Package-base", "base", "1", "pkg:deb/debian/base@1?arch=amd64&distro=debian-12.15"), + package("SPDXRef-Package-extension", "extension", "2", "pkg:deb/debian/extension@2?arch=amd64&distro=debian-12.15"), + package("SPDXRef-Package-build-only", "build-only", "3", "pkg:deb/debian/build-only@3?arch=amd64&distro=debian-12.15"), + ], + "files": [ + {"SPDXID": "SPDXRef-File-base", "fileName": "usr/lib/base.so", "checksums": checksum("base")}, + {"SPDXID": "SPDXRef-File-extension", "fileName": "usr/lib/postgresql/ext.so", "checksums": checksum("extension")}, + {"SPDXID": "SPDXRef-File-build-only", "fileName": "usr/bin/cc", "checksums": checksum("build-only")}, + ], + "relationships": [ + {"spdxElementId": "SPDXRef-Package-base", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-File-base"}, + {"spdxElementId": "SPDXRef-Package-extension", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-File-extension"}, + {"spdxElementId": "SPDXRef-Package-build-only", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-File-build-only"}, + {"spdxElementId": "SPDXRef-Package-base", "relationshipType": "DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-Package-extension"}, + ], + } + + +def inventory(*entries): + return {"files": [{"name": name, "algorithm": "sha256", "value": digest} for name, digest in entries]} + + +class ComposeTest(unittest.TestCase): + def test_composes_only_shipped_files_and_owned_packages(self): + output = compose( + builder_document(), + extension_name="plr", + final_inventory=inventory( + ("lib/ext.so", "extension"), + ("generated/artifact", "generated"), + ), + platform="linux/amd64", + ) + self.assertEqual(output["name"], "plr-sbom") + self.assertEqual( + {record["name"] for record in output["packages"]}, + {"extension", "debian", "plr-extension-artifacts"}, + ) + self.assertEqual( + [record["fileName"] for record in output["files"]], + ["generated/artifact", "lib/ext.so"], + ) + self.assertNotIn("build-only", json.dumps(output)) + self.assertFalse("subject" in output) + + def test_license_files_are_directly_mapped_to_the_named_package(self): + document = builder_document() + document["packages"].append( + package("SPDXRef-Package-copyright", "libgomp1", "1", "pkg:deb/debian/libgomp1@1?arch=amd64&distro=debian-12.15") + ) + document["files"].append({ + "SPDXID": "SPDXRef-File-copyright", + "fileName": "usr/share/doc/libgomp1/copyright", + "checksums": checksum("copyright"), + }) + document["relationships"].append({ + "spdxElementId": "SPDXRef-Package-copyright", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-File-copyright", + }) + output = compose( + document, + extension_name="plr", + final_inventory=inventory(("licenses/libgomp1/copyright", "copyright")), + platform="linux/amd64", + scancode_report={"files": [{ + "path": "licenses/libgomp1/copyright", + "license_detections": [{"license_expression_spdx": "GPL-2.0-only"}], + }]}, + ) + file_record = output["files"][0] + package_record = next(item for item in output["packages"] if item["name"] == "libgomp1") + self.assertEqual(file_record["licenseInfoInFiles"], ["GPL-2.0-only"]) + self.assertEqual(package_record["licenseDeclared"], "GPL-2.0-only") + + def test_unmatched_file_licenses_are_combined_on_synthetic_package(self): + output = compose( + builder_document(), + extension_name="demo", + final_inventory=inventory( + ("licenses/vendor/copyright", "vendor-license"), + ("licenses/other/copyright", "other-license"), + ), + platform="linux/amd64", + scancode_report={"files": [ + { + "path": "licenses/vendor/copyright", + "license_detections": [{"license_expression_spdx": "MIT"}], + }, + { + "path": "licenses/other/copyright", + "license_detections": [{ + "license_expression_spdx": "Apache-2.0 OR BSD-2-Clause", + }], + }, + ]}, + ) + synthetic = next( + item for item in output["packages"] + if item["name"] == "demo-extension-artifacts" + ) + self.assertEqual(synthetic["licenseInfoFromFiles"], [ + "Apache-2.0 OR BSD-2-Clause", "MIT", + ]) + self.assertEqual( + synthetic["licenseDeclared"], + "(Apache-2.0 OR BSD-2-Clause) AND MIT", + ) + + def test_platform_documents_are_deterministic_and_isolated(self): + first = compose( + builder_document(), extension_name="plr", + final_inventory=inventory(("lib/ext.so", "extension")), platform="linux/amd64", + evidence={"builderSha256": "abc"}, + ) + second = compose( + builder_document(), extension_name="plr", + final_inventory=inventory(("lib/ext.so", "extension")), platform="linux/arm64", + evidence={"builderSha256": "abc"}, + ) + self.assertNotEqual(first["documentNamespace"], second["documentNamespace"]) + self.assertEqual(first, compose( + builder_document(), extension_name="plr", + final_inventory=inventory(("lib/ext.so", "extension")), platform="linux/amd64", + evidence={"builderSha256": "abc"}, + )) + + def test_namespace_changes_with_content_and_is_stable_when_reapplied(self): + first = compose( + builder_document(), extension_name="demo", + final_inventory=inventory(("lib/ext.so", "extension")), platform="linux/amd64", + ) + changed = compose( + builder_document(), extension_name="demo", + final_inventory=inventory(("lib/ext.so", "different")), platform="linux/amd64", + ) + self.assertNotEqual(first["documentNamespace"], changed["documentNamespace"]) + original_namespace = first["documentNamespace"] + set_document_namespace(first, "demo", "linux/amd64") + self.assertEqual(first["documentNamespace"], original_namespace) + first["comment"] = "Downstream augmentation" + set_document_namespace(first, "demo", "linux/amd64") + self.assertNotEqual(first["documentNamespace"], original_namespace) + + def test_malformed_final_inventory_fails(self): + with self.assertRaises(ValueError): + final_inventory_files({"files": [{"name": "lib/ext.so", "checksums": []}]}, Path("inventory")) + + def test_builder_wrapper_is_accepted_only_as_legacy_input(self): + wrapped = { + "predicateType": "https://spdx.dev/Document", + "predicate": builder_document(), + "subject": [{"name": "lib/ext.so", "digest": {"sha256": "extension"}}], + } + output = compose(wrapped, extension_name="plr") + self.assertEqual(output["name"], "plr-sbom") + + +if __name__ == "__main__": + unittest.main() diff --git a/sbom-generator/tests/test_generator.py b/sbom-generator/tests/test_generator.py new file mode 100644 index 00000000..fa69349b --- /dev/null +++ b/sbom-generator/tests/test_generator.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parents[1])) + +from generator import final_inventory, infer_platform, scan_licenses, statement_for # noqa: E402 + + +class GeneratorTest(unittest.TestCase): + def test_inventory_hashes_regular_and_symlink_files_without_following_links(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "lib").mkdir() + payload = b"payload" + (root / "lib" / "ext.so").write_bytes(payload) + (root / "lib" / "alias.so").symlink_to("ext.so") + (root / "outside").symlink_to("/tmp", target_is_directory=True) + output = final_inventory(root) + records = {record["name"]: record for record in output["files"]} + self.assertEqual(records["lib/ext.so"]["value"], hashlib.sha256(payload).hexdigest()) + self.assertEqual(records["lib/alias.so"]["kind"], "symlink") + self.assertNotEqual(records["lib/alias.so"]["value"], records["lib/ext.so"]["value"]) + self.assertEqual(records["outside"]["kind"], "symlink") + + def test_platform_comes_from_builder_package_architecture(self): + document = {"packages": [{"externalRefs": [{ + "referenceType": "purl", + "referenceLocator": "pkg:deb/debian/base@1?arch=amd64&distro=debian-12.15", + }]}]} + self.assertEqual(infer_platform(document), "linux/amd64") + + def test_platform_ambiguity_is_rejected(self): + document = {"packages": [{"externalRefs": [ + {"referenceType": "purl", "referenceLocator": "pkg:deb/debian/a@1?arch=amd64"}, + {"referenceType": "purl", "referenceLocator": "pkg:deb/debian/b@1?arch=arm64"}, + ]}]} + with self.assertRaises(RuntimeError): + infer_platform(document) + + def test_output_is_an_spdx_intoto_statement(self): + predicate = {"SPDXID": "SPDXRef-DOCUMENT", "name": "demo"} + statement = statement_for(predicate) + self.assertEqual(statement["_type"], "https://in-toto.io/Statement/v1") + self.assertEqual(statement["predicateType"], "https://spdx.dev/Document") + self.assertEqual(statement["predicate"], predicate) + self.assertEqual(statement["subject"], []) + + def test_license_files_are_split_before_scancode_and_paths_are_collapsed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "licenses").mkdir() + (root / "licenses" / "copyright").write_text("license text") + temporary = root / "temporary" + temporary.mkdir() + commands = [] + + def fake_scanner(command, **_kwargs): + commands.append(command) + if command[0] == "csplit": + prefix = Path(command[command.index("-f") + 1]) + prefix.with_name("license-00").write_text("license text") + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + output = Path(command[command.index("--json") + 1]) + output.write_text(json.dumps({ + "files": [{ + "type": "file", + "path": "licenses/copyright/license-00", + "license_detections": [{ + "matches": [{"from_file": "licenses/copyright/license-00"}], + "license_expression_spdx": "MIT", + }], + }], + })) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + with patch("generator.shutil.which", return_value="scancode"), patch( + "generator.subprocess.run", side_effect=fake_scanner + ): + report = scan_licenses(root, temporary) + + self.assertEqual(report["files"][0]["path"], "licenses/copyright") + self.assertEqual( + report["files"][0]["license_detections"][0]["matches"][0]["from_file"], + "licenses/copyright", + ) + split_command = next(command for command in commands if command[0] == "csplit") + self.assertIn("/^License:/", split_command) + self.assertIn("{*}", split_command) + + +if __name__ == "__main__": + unittest.main() diff --git a/sbom-generator/tests/test_hooks.py b/sbom-generator/tests/test_hooks.py new file mode 100644 index 00000000..6f5381f9 --- /dev/null +++ b/sbom-generator/tests/test_hooks.py @@ -0,0 +1,78 @@ +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parents[1])) + +import generator +from hooks import HookContext, run_augmentation_hook +from test_compose import builder_document + + +def write_hook(builder, source): + path = builder / "usr/local/share/cnpg-sbom/augment_spdx.py" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + + +class HookTest(unittest.TestCase): + def test_absent_hook_preserves_document(self): + document = {"packages": []} + with tempfile.TemporaryDirectory() as directory: + context = HookContext("demo", "linux/amd64", Path(directory), Path(directory), {}) + self.assertIs(run_augmentation_hook(document, context), document) + + def test_hook_receives_full_builder_evidence_and_output_is_wrapped(self): + evidence = builder_document() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source, extras, destination = root / "source", root / "extras", root / "output" + source.mkdir() + builder = extras / "sbom-builder" + builder.mkdir(parents=True) + destination.mkdir() + (source / "artifact").write_text("payload") + write_hook(builder, ''' +from hooks import HookContext + +def augment_spdx(document, context: HookContext): + assert context.api_version == 1 + assert context.platform == "linux/amd64" + assert context.builder_path.is_dir() + assert context.final_path.is_dir() + assert "build-only" not in {p["name"] for p in document["packages"]} + assert "build-only" in {p["name"] for p in context.builder_document["packages"]} + assert document["files"][0]["SPDXID"] + assert document["files"][0]["checksums"] + return {**document, "comment": "downstream dependency version and license information"} +''') + with patch.dict(os.environ, { + "BUILDKIT_SCAN_SOURCE": str(source), + "BUILDKIT_SCAN_SOURCE_EXTRAS": str(extras), + "BUILDKIT_SCAN_DESTINATION": str(destination), + "BUILDKIT_BUILDER_SPDX": "", + "SBOM_TARGET_PLATFORM": "linux/amd64", + }), patch.object(generator, "scan_builder", return_value=evidence), \ + patch.object(generator, "tool_version", return_value="test"): + statement = json.loads(generator.generate().read_text()) + self.assertEqual(statement["predicate"]["comment"], + "downstream dependency version and license information") + self.assertEqual(statement["subject"], []) + self.assertEqual([f["fileName"] for f in statement["predicate"]["files"]], ["artifact"]) + + def test_hook_errors_are_not_silently_ignored(self): + sources = ( + "def augment_spdx(document, context):\n raise ValueError('evidence missing')\n", + "# Missing entry point\n", + "invalid python syntax!\n", + ) + for source in sources: + with self.subTest(source=source), tempfile.TemporaryDirectory() as directory: + builder = Path(directory) + write_hook(builder, source) + with self.assertRaisesRegex(RuntimeError, "SBOM hook"): + run_augmentation_hook({}, HookContext("demo", "linux/amd64", builder, builder, {})) From bb6032230b611ca347f5cb604cac7c5ddb71ce07 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:21:03 +0000 Subject: [PATCH 02/24] Use cnpg-extensions SBOM generator image --- .github/workflows/bake_targets.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 37bb8b5e..76c91e31 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -60,8 +60,8 @@ jobs: environment: testing registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} - # renovate: datasource=docker depName=ghcr.io/ardentperf/cnpg-sbom-generator - sbom_generator: ghcr.io/ardentperf/cnpg-sbom-generator:latest + # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator + sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . From 9d994db9664f3c9b1750da618369118d5e44fdc1 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:22:13 +0000 Subject: [PATCH 03/24] Use explicit CNPG Extensions generator image path --- .github/workflows/sbom-generator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sbom-generator.yml b/.github/workflows/sbom-generator.yml index 2319fe37..e65eb0b5 100644 --- a/.github/workflows/sbom-generator.yml +++ b/.github/workflows/sbom-generator.yml @@ -52,7 +52,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} run: | - image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/cnpg-sbom-generator" + image="ghcr.io/cnpg-extensions/cnpg-sbom-generator" options=() if [[ "$EVENT_NAME" == push ]]; then options+=(--push); fi docker buildx build \ From 2ebef092a1c7268609787773df8457502ecf1066 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:41:03 +0000 Subject: [PATCH 04/24] Allow manual SBOM generator publishing for tests Add a workflow_dispatch publish input so the generator image can be pushed to GHCR on demand. Keep pull-request builds non-publishing and preserve local registry testing. Use the custom generator only on main or in explicit local mode so feature-branch extension CI does not depend on an unpublished image. --- .github/workflows/bake_targets.yml | 30 +++++++++++++++++++------ .github/workflows/sbom-generator.yml | 33 ++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 76c91e31..27c7549b 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -7,6 +7,11 @@ on: description: "The PostgreSQL extension to build (directory name)" required: true type: string + local: + description: "Use the local registry and skip GitHub-only side effects" + required: false + default: false + type: boolean secrets: SNYK_TOKEN: required: false @@ -32,6 +37,7 @@ jobs: persist-credentials: false - name: Log in to the GitHub Container registry + if: ${{ inputs.local != true }} uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io @@ -39,34 +45,39 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU + if: ${{ inputs.local != true }} uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 with: platforms: 'linux/arm64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 + with: + driver-opts: ${{ inputs.local && 'network=host' || '' }} - name: Expose builder stage to SBOM generator + if: ${{ inputs.local == true || github.ref == 'refs/heads/main' }} env: EXTENSION: ${{ inputs.extension_name }} run: | sed -i '2i ARG BUILDKIT_SBOM_SCAN_STAGE=builder' "$EXTENSION/Dockerfile" - name: Build and push - uses: docker/bake-action@018cb6412ab401ebaa809aa5f85966b74628600f # v7 + uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7 id: build env: BUILDX_METADATA_PROVENANCE: disabled environment: testing - registry: ghcr.io/${{ github.repository_owner }} + registry: ${{ inputs.local && '127.0.0.1:5000' || format('ghcr.io/{0}', github.repository_owner) }} revision: ${{ github.sha }} # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator - sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest + sbom_generator: ${{ inputs.local && '127.0.0.1:5000/cnpg-sbom-generator:latest' || (github.ref == 'refs/heads/main' && 'ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest' || '') }} with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . files: ./docker-bake.hcl,./${{ inputs.extension_name }}/metadata.hcl push: true + set: ${{ inputs.local && '*.platform=linux/amd64' || '' }} # From bake's metadata, extract each unique tag (e.g. the ones with the timestamp) - name: Generated images @@ -78,11 +89,13 @@ jobs: # Even if we're testing we sign the images, so we can push them to production later if that's required - name: Install cosign + if: ${{ inputs.local != true }} uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 # See https://github.blog/security/supply-chain-security/safeguard-container-signing-capability-actions/ # and https://github.com/actions/starter-workflows/blob/main/ci/docker-publish.yml for more details on # how to use cosign. - name: Sign images + if: ${{ inputs.local != true }} env: BUILD_METADATA: ${{ steps.build.outputs.metadata }} run: | @@ -91,6 +104,7 @@ jobs: security: name: Security checks + if: ${{ inputs.local != true }} runs-on: ubuntu-24.04 permissions: contents: read @@ -101,7 +115,7 @@ jobs: strategy: fail-fast: false matrix: - image: ${{fromJson(needs.testbuild.outputs.images)}} + image: ${{ inputs.local && fromJSON('["local"]') || fromJSON(needs.testbuild.outputs.images) }} steps: - name: Checkout Code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -119,6 +133,7 @@ jobs: generate-smoke-test-matrix: name: Generate matrix for smoke tests + if: ${{ inputs.local != true }} runs-on: ubuntu-24.04 permissions: {} outputs: @@ -138,6 +153,7 @@ jobs: smoke-test: name: Smoke test + if: ${{ inputs.local != true }} runs-on: ubuntu-24.04 permissions: contents: read @@ -148,8 +164,8 @@ jobs: strategy: fail-fast: false matrix: - image: ${{fromJson(needs.testbuild.outputs.images)}} - cnpg: ${{fromJson(needs.generate-smoke-test-matrix.outputs.versions)}} + image: ${{ inputs.local && fromJSON('["local"]') || fromJSON(needs.testbuild.outputs.images) }} + cnpg: ${{ inputs.local && fromJSON('["local"]') || fromJSON(needs.generate-smoke-test-matrix.outputs.versions) }} steps: - name: Checkout Code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -190,7 +206,7 @@ jobs: copytoproduction: name: Copy images to production - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ inputs.local != true && github.ref == 'refs/heads/main' }} runs-on: ubuntu-24.04 needs: - testbuild diff --git a/.github/workflows/sbom-generator.yml b/.github/workflows/sbom-generator.yml index e65eb0b5..b9b4f1b6 100644 --- a/.github/workflows/sbom-generator.yml +++ b/.github/workflows/sbom-generator.yml @@ -10,6 +10,18 @@ on: paths: - 'sbom-generator/**' - '.github/workflows/sbom-generator.yml' + workflow_dispatch: + inputs: + local: + description: "Push to the local test registry instead of GHCR" + required: false + default: false + type: boolean + publish: + description: "Publish the generator image to GHCR (manual testing only)" + required: false + default: false + type: boolean permissions: {} @@ -33,15 +45,18 @@ jobs: run: python3 -m unittest discover -s sbom-generator/tests -p 'test_*.py' - name: Set up QEMU + if: ${{ inputs.local != true }} uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 with: platforms: linux/arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + with: + driver-opts: ${{ inputs.local && 'network=host' || '' }} - name: Log in to GHCR - if: github.event_name == 'push' + if: ${{ (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish == true)) && inputs.local != true }} uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io @@ -51,12 +66,22 @@ jobs: - name: Build generator env: EVENT_NAME: ${{ github.event_name }} + LOCAL: ${{ inputs.local }} + PUBLISH: ${{ inputs.publish }} run: | - image="ghcr.io/cnpg-extensions/cnpg-sbom-generator" + registry="ghcr.io/cnpg-extensions" + if [[ "$LOCAL" == true ]]; then + registry="127.0.0.1:5000" + fi + image="$registry/cnpg-sbom-generator" + platforms="linux/amd64,linux/arm64" + if [[ "$LOCAL" == true ]]; then + platforms="linux/amd64" + fi options=() - if [[ "$EVENT_NAME" == push ]]; then options+=(--push); fi + if [[ "$EVENT_NAME" == push || "$PUBLISH" == true || "$LOCAL" == true ]]; then options+=(--push); fi docker buildx build \ - --platform linux/amd64,linux/arm64 \ + --platform "$platforms" \ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "$image:latest" \ --tag "$image:sha-${GITHUB_SHA}" \ From a4d4832fed05442a9889c14880ce22576939e254 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:45:58 +0000 Subject: [PATCH 05/24] Remove local Act support from CI workflows Keep the workflow_dispatch publish switch for explicit GHCR testing, but remove the local registry, single-platform, network, and skipped-side-effect paths that were only needed by local Act runs. --- .github/workflows/bake_targets.yml | 29 +++++++--------------------- .github/workflows/sbom-generator.yml | 25 ++++-------------------- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 27c7549b..67bef2ea 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -7,11 +7,6 @@ on: description: "The PostgreSQL extension to build (directory name)" required: true type: string - local: - description: "Use the local registry and skip GitHub-only side effects" - required: false - default: false - type: boolean secrets: SNYK_TOKEN: required: false @@ -37,7 +32,6 @@ jobs: persist-credentials: false - name: Log in to the GitHub Container registry - if: ${{ inputs.local != true }} uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io @@ -45,18 +39,15 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - if: ${{ inputs.local != true }} uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 with: platforms: 'linux/arm64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 - with: - driver-opts: ${{ inputs.local && 'network=host' || '' }} - name: Expose builder stage to SBOM generator - if: ${{ inputs.local == true || github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} env: EXTENSION: ${{ inputs.extension_name }} run: | @@ -68,16 +59,15 @@ jobs: env: BUILDX_METADATA_PROVENANCE: disabled environment: testing - registry: ${{ inputs.local && '127.0.0.1:5000' || format('ghcr.io/{0}', github.repository_owner) }} + registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator - sbom_generator: ${{ inputs.local && '127.0.0.1:5000/cnpg-sbom-generator:latest' || (github.ref == 'refs/heads/main' && 'ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest' || '') }} + sbom_generator: ${{ github.ref == 'refs/heads/main' && 'ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest' || '' }} with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . files: ./docker-bake.hcl,./${{ inputs.extension_name }}/metadata.hcl push: true - set: ${{ inputs.local && '*.platform=linux/amd64' || '' }} # From bake's metadata, extract each unique tag (e.g. the ones with the timestamp) - name: Generated images @@ -89,13 +79,11 @@ jobs: # Even if we're testing we sign the images, so we can push them to production later if that's required - name: Install cosign - if: ${{ inputs.local != true }} uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 # See https://github.blog/security/supply-chain-security/safeguard-container-signing-capability-actions/ # and https://github.com/actions/starter-workflows/blob/main/ci/docker-publish.yml for more details on # how to use cosign. - name: Sign images - if: ${{ inputs.local != true }} env: BUILD_METADATA: ${{ steps.build.outputs.metadata }} run: | @@ -104,7 +92,6 @@ jobs: security: name: Security checks - if: ${{ inputs.local != true }} runs-on: ubuntu-24.04 permissions: contents: read @@ -115,7 +102,7 @@ jobs: strategy: fail-fast: false matrix: - image: ${{ inputs.local && fromJSON('["local"]') || fromJSON(needs.testbuild.outputs.images) }} + image: ${{fromJson(needs.testbuild.outputs.images)}} steps: - name: Checkout Code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -133,7 +120,6 @@ jobs: generate-smoke-test-matrix: name: Generate matrix for smoke tests - if: ${{ inputs.local != true }} runs-on: ubuntu-24.04 permissions: {} outputs: @@ -153,7 +139,6 @@ jobs: smoke-test: name: Smoke test - if: ${{ inputs.local != true }} runs-on: ubuntu-24.04 permissions: contents: read @@ -164,8 +149,8 @@ jobs: strategy: fail-fast: false matrix: - image: ${{ inputs.local && fromJSON('["local"]') || fromJSON(needs.testbuild.outputs.images) }} - cnpg: ${{ inputs.local && fromJSON('["local"]') || fromJSON(needs.generate-smoke-test-matrix.outputs.versions) }} + image: ${{fromJson(needs.testbuild.outputs.images)}} + cnpg: ${{fromJson(needs.generate-smoke-test-matrix.outputs.versions)}} steps: - name: Checkout Code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -206,7 +191,7 @@ jobs: copytoproduction: name: Copy images to production - if: ${{ inputs.local != true && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} runs-on: ubuntu-24.04 needs: - testbuild diff --git a/.github/workflows/sbom-generator.yml b/.github/workflows/sbom-generator.yml index b9b4f1b6..d4ec03b3 100644 --- a/.github/workflows/sbom-generator.yml +++ b/.github/workflows/sbom-generator.yml @@ -12,11 +12,6 @@ on: - '.github/workflows/sbom-generator.yml' workflow_dispatch: inputs: - local: - description: "Push to the local test registry instead of GHCR" - required: false - default: false - type: boolean publish: description: "Publish the generator image to GHCR (manual testing only)" required: false @@ -45,18 +40,15 @@ jobs: run: python3 -m unittest discover -s sbom-generator/tests -p 'test_*.py' - name: Set up QEMU - if: ${{ inputs.local != true }} uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 with: platforms: linux/arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - with: - driver-opts: ${{ inputs.local && 'network=host' || '' }} - name: Log in to GHCR - if: ${{ (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish == true)) && inputs.local != true }} + if: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish == true) }} uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io @@ -66,22 +58,13 @@ jobs: - name: Build generator env: EVENT_NAME: ${{ github.event_name }} - LOCAL: ${{ inputs.local }} PUBLISH: ${{ inputs.publish }} run: | - registry="ghcr.io/cnpg-extensions" - if [[ "$LOCAL" == true ]]; then - registry="127.0.0.1:5000" - fi - image="$registry/cnpg-sbom-generator" - platforms="linux/amd64,linux/arm64" - if [[ "$LOCAL" == true ]]; then - platforms="linux/amd64" - fi + image="ghcr.io/cnpg-extensions/cnpg-sbom-generator" options=() - if [[ "$EVENT_NAME" == push || "$PUBLISH" == true || "$LOCAL" == true ]]; then options+=(--push); fi + if [[ "$EVENT_NAME" == push || "$PUBLISH" == true ]]; then options+=(--push); fi docker buildx build \ - --platform "$platforms" \ + --platform linux/amd64,linux/arm64 \ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "$image:latest" \ --tag "$image:sha-${GITHUB_SHA}" \ From f37a184b1a2ca4d00ff2ae5646f91c5803686252 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:44:57 +0000 Subject: [PATCH 06/24] Separate SBOM and extension build triggers --- .github/workflows/bake.yml | 2 ++ .github/workflows/bake_targets.yml | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index 770be422..2f789a81 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -5,6 +5,8 @@ on: paths-ignore: - '.github/workflows/pgrx.yml' - '.github/workflows/pgrx_targets.yml' + - '.github/workflows/sbom-generator.yml' + - 'sbom-generator/**' workflow_dispatch: inputs: extension_name: diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 67bef2ea..20316932 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -47,7 +47,6 @@ jobs: uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 - name: Expose builder stage to SBOM generator - if: ${{ github.ref == 'refs/heads/main' }} env: EXTENSION: ${{ inputs.extension_name }} run: | @@ -62,7 +61,7 @@ jobs: registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator - sbom_generator: ${{ github.ref == 'refs/heads/main' && 'ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest' || '' }} + sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:cbd49cfd14fd66e7ac3c3c25feea70f9ad7f62a3b4faf1c7cbea1e21602ed9bb with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . From 0e72b929030d433e453b2d860feb104aa474fdd5 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:51:51 +0000 Subject: [PATCH 07/24] Add SBOM generator progress diagnostics Identify the generator with its version and source repository, and emit phase timing plus scanner heartbeats so slow BuildKit SBOM runs are diagnosable. --- sbom-generator/README.md | 9 +++ sbom-generator/compose.py | 10 ++- sbom-generator/generator.py | 99 +++++++++++++++++++++----- sbom-generator/tests/test_compose.py | 20 ++++++ sbom-generator/tests/test_generator.py | 56 ++++++++++++--- 5 files changed, 167 insertions(+), 27 deletions(-) mode change 100755 => 100644 sbom-generator/compose.py diff --git a/sbom-generator/README.md b/sbom-generator/README.md index c0ab75ac..3f1b78e9 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -149,6 +149,15 @@ payload; BuildKit supplies the final image subject, attestation manifest, and image index. A multi-platform index therefore contains one image and one combined provenance/SPDX attestation for each platform. +The SPDX creation metadata identifies the generator as +`Tool: cnpg-sbom-generator-1`. Its document annotation retains the generator +name, version, and source repository URL; the immutable generator image digest +selected by the workflow provides the stronger reproducibility boundary. + +During generation, BuildKit logs show phase start/completion and elapsed time. +Long Syft and ScanCode subprocesses emit a heartbeat every 30 seconds, and +license-file preparation reports its file and chunk counts. + The examples below use the H3 image produced by this repository. Replace `INDEX_DIGEST` with the immutable index digest returned by `docker buildx imagetools inspect`; the placeholder is intentional because a diff --git a/sbom-generator/compose.py b/sbom-generator/compose.py old mode 100755 new mode 100644 index 0018319e..dae0c346 --- a/sbom-generator/compose.py +++ b/sbom-generator/compose.py @@ -21,6 +21,9 @@ EXTENSION_PACKAGE_ID = "SPDXRef-Package-extension-payload" +GENERATOR_NAME = "cnpg-sbom-generator" +GENERATOR_VERSION = "1" +GENERATOR_REPOSITORY = "https://github.com/cnpg-extensions/postgres-extensions-containers" LICENSE_REF = re.compile(r"LicenseRef-[A-Za-z0-9][A-Za-z0-9.-]*") LICENSE_OPERATOR = re.compile(r"\s+(?:AND|OR|WITH)\s+") @@ -500,13 +503,14 @@ def add_synthetic_file(record: dict[str, str], owner: str | None = None) -> None raise ValueError(f"unsupported target platform: {platform!r}") creation_info = output.setdefault("creationInfo", {}) creators = list(creation_info.get("creators", [])) - generator_creator = "Tool: cnpg-sbom-generator" + generator_creator = f"Tool: {GENERATOR_NAME}-{GENERATOR_VERSION}" if generator_creator not in creators: creators.append(generator_creator) creation_info["creators"] = creators metadata = { - "generator": "cnpg-sbom-generator", - "generatorVersion": "1", + "generator": GENERATOR_NAME, + "generatorVersion": GENERATOR_VERSION, + "generatorRepository": GENERATOR_REPOSITORY, "platform": platform, "evidence": evidence or {}, } diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 927159f4..47855a3b 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -19,6 +19,7 @@ import subprocess import sys import tempfile +import time from pathlib import Path from typing import Any, Sequence @@ -35,6 +36,7 @@ INTOTO_STATEMENT_TYPE = "https://in-toto.io/Statement/v1" SPDX_PREDICATE_TYPE = "https://spdx.dev/Document" +PROGRESS_INTERVAL_SECONDS = 30 def require_directory(value: str | None, variable: str) -> Path: @@ -119,19 +121,58 @@ def add_symlink(path: Path) -> None: return {"files": records} -def run_json_command(command: Sequence[str], output: Path) -> dict[str, Any]: +def progress(message: str) -> None: + print(f"sbom-generator: {message}", file=sys.stderr, flush=True) + + +def run_command_with_progress( + command: Sequence[str], label: str +) -> subprocess.CompletedProcess: + """Run a scanner while keeping long-running phases visible in BuildKit logs.""" + + progress(f"{label} started") + started = time.monotonic() try: - subprocess.run( - [*command, f"spdx-json={output}"], - check=True, - capture_output=True, + process = subprocess.Popen( + list(command), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, ) + except FileNotFoundError: + raise + + while True: + try: + stdout, stderr = process.communicate(timeout=PROGRESS_INTERVAL_SECONDS) + break + except subprocess.TimeoutExpired: + progress( + f"{label} still running " + f"({time.monotonic() - started:.0f}s elapsed)" + ) + + result = subprocess.CompletedProcess( + list(command), process.returncode, stdout, stderr + ) + elapsed = time.monotonic() - started + if result.returncode: + detail = (result.stderr or result.stdout or "scanner failed").strip() + raise RuntimeError(f"{' '.join(command)} failed: {detail}") + progress(f"{label} complete in {elapsed:.1f}s") + return result + + +def run_json_command( + command: Sequence[str], output: Path, label: str +) -> dict[str, Any]: + try: + run_command_with_progress( + [*command, f"spdx-json={output}"], + label, + ) except FileNotFoundError as error: raise RuntimeError(f"required scanner is unavailable: {command[0]}") from error - except subprocess.CalledProcessError as error: - detail = (error.stderr or error.stdout or "scanner failed").strip() - raise RuntimeError(f"{' '.join(command)} failed: {detail}") from error try: with output.open(encoding="utf-8") as stream: document = json.load(stream) @@ -152,21 +193,26 @@ def scan_builder(builder: Path, temporary: Path) -> dict[str, Any]: document = json.load(stream) if not isinstance(document, dict): raise RuntimeError("BUILDKIT_BUILDER_SPDX is not a JSON object") + progress("using supplied builder SPDX fixture") return document syft = shutil.which("syft") if not syft: raise RuntimeError("syft is required to scan the mounted builder stage") output = temporary / "builder.spdx.json" - return run_json_command( + document = run_json_command( [syft, f"dir:{builder}", "--scope", "all-layers", "--quiet", "--output"], output, + "Syft builder scan", ) + progress(f"Syft found {len(document.get('packages', []))} builder packages") + return document def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: licenses = final_root / "licenses" if not licenses.exists(): + progress("no /licenses directory; skipping ScanCode") return {"files": []} scancode = shutil.which("scancode") if not scancode: @@ -174,7 +220,8 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: scan_root = prepare_license_scan_root(final_root, temporary) output = temporary / "scancode.json" try: - subprocess.run( + input_files = sum(1 for path in scan_root.rglob("*") if path.is_file()) + run_command_with_progress( [ scancode, "--license", @@ -183,13 +230,10 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: str(output), str(scan_root), ], - check=True, - capture_output=True, - text=True, + f"ScanCode license scan ({input_files} input files)", ) - except subprocess.CalledProcessError as error: - detail = (error.stderr or error.stdout or "scancode failed").strip() - raise RuntimeError(f"scancode failed: {detail}") from error + except FileNotFoundError as error: + raise RuntimeError("scancode is required when the final payload has /licenses") from error try: with output.open(encoding="utf-8") as stream: report = json.load(stream) @@ -198,6 +242,7 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: if not isinstance(report, dict): raise RuntimeError("scancode output is not a JSON object") normalize_scancode_report_paths(report, scan_root, final_root) + progress(f"ScanCode reported {len(report.get('files', []))} files") return report @@ -211,9 +256,13 @@ def prepare_license_scan_root(final_root: Path, temporary: Path) -> Path: licenses = final_root / "licenses" scan_root = temporary / "license-chunks" / "licenses" + started = time.monotonic() + license_file_count = 0 + chunk_count = 0 for license_file in sorted(licenses.rglob("*")): if license_file.is_symlink() or not license_file.is_file(): continue + license_file_count += 1 relative = license_file.relative_to(licenses) chunk_directory = scan_root / relative chunk_directory.mkdir(parents=True, exist_ok=True) @@ -239,6 +288,19 @@ def prepare_license_scan_root(final_root: Path, temporary: Path) -> Path: if result.returncode: detail = (result.stderr or result.stdout or "csplit failed").strip() raise RuntimeError(f"csplit failed for {license_file}: {detail}") + chunk_count += sum( + 1 for path in chunk_directory.glob("license-*") if path.is_file() + ) + if license_file_count % 100 == 0: + progress( + f"prepared {license_file_count} license files " + f"({chunk_count} ScanCode chunks, " + f"{time.monotonic() - started:.1f}s elapsed)" + ) + progress( + f"prepared {license_file_count} license files into {chunk_count} " + f"ScanCode chunks in {time.monotonic() - started:.1f}s" + ) return scan_root @@ -341,12 +403,16 @@ def generate() -> Path: raise RuntimeError(f"scanner output directory must be empty: {destination}") extension_name = os.getenv("SBOM_EXTENSION_NAME", "extension") + progress(f"starting SBOM for {extension_name}") with tempfile.TemporaryDirectory(prefix="cnpg-sbom-") as temporary_name: temporary = Path(temporary_name) builder_document = scan_builder(builder, temporary) platform = infer_platform(builder_document) + progress(f"target platform: {platform}") inventory = final_inventory(source) + progress(f"final payload inventory contains {len(inventory['files'])} files") report = scan_licenses(source, temporary) + progress("composing SPDX document") evidence = { "builderSha256": sha256_file(Path(os.getenv("BUILDKIT_BUILDER_SPDX"))) if os.getenv("BUILDKIT_BUILDER_SPDX") @@ -379,6 +445,7 @@ def generate() -> Path: set_document_namespace(predicate, extension_name, platform) statement = statement_for(predicate) output = destination / "final-payload.spdx.json" + progress("writing SPDX attestation") with output.open("w", encoding="utf-8") as stream: json.dump(statement, stream, indent=2, sort_keys=True) stream.write("\n") diff --git a/sbom-generator/tests/test_compose.py b/sbom-generator/tests/test_compose.py index de0b2830..f56e2878 100644 --- a/sbom-generator/tests/test_compose.py +++ b/sbom-generator/tests/test_compose.py @@ -88,6 +88,26 @@ def test_composes_only_shipped_files_and_owned_packages(self): self.assertNotIn("build-only", json.dumps(output)) self.assertFalse("subject" in output) + def test_generator_metadata_identifies_version_and_repository(self): + output = compose( + builder_document(), + extension_name="plr", + final_inventory=inventory(("lib/ext.so", "extension")), + platform="linux/amd64", + ) + generator_annotation = next( + annotation for annotation in output["annotations"] + if annotation["annotator"] == "Tool: cnpg-sbom-generator-1" + ) + metadata = json.loads(generator_annotation["comment"]) + self.assertIn("Tool: cnpg-sbom-generator-1", output["creationInfo"]["creators"]) + self.assertEqual(metadata["generator"], "cnpg-sbom-generator") + self.assertEqual(metadata["generatorVersion"], "1") + self.assertEqual( + metadata["generatorRepository"], + "https://github.com/cnpg-extensions/postgres-extensions-containers", + ) + def test_license_files_are_directly_mapped_to_the_named_package(self): document = builder_document() document["packages"].append( diff --git a/sbom-generator/tests/test_generator.py b/sbom-generator/tests/test_generator.py index fa69349b..d29903d5 100644 --- a/sbom-generator/tests/test_generator.py +++ b/sbom-generator/tests/test_generator.py @@ -11,7 +11,13 @@ sys.path.insert(0, str(Path(__file__).parents[1])) -from generator import final_inventory, infer_platform, scan_licenses, statement_for # noqa: E402 +from generator import ( # noqa: E402 + final_inventory, + infer_platform, + run_command_with_progress, + scan_licenses, + statement_for, +) class GeneratorTest(unittest.TestCase): @@ -53,6 +59,31 @@ def test_output_is_an_spdx_intoto_statement(self): self.assertEqual(statement["predicate"], predicate) self.assertEqual(statement["subject"], []) + def test_long_running_scanner_reports_a_heartbeat(self): + messages = [] + + class FakeProcess: + returncode = 0 + + def __init__(self): + self.calls = 0 + + def communicate(self, timeout=None): + self.calls += 1 + if self.calls == 1: + raise subprocess.TimeoutExpired(["scanner"], timeout) + return "", "" + + with patch("generator.subprocess.Popen", return_value=FakeProcess()), patch( + "generator.progress", side_effect=messages.append + ): + result = run_command_with_progress(["scanner"], "test scanner") + + self.assertEqual(result.returncode, 0) + self.assertEqual(messages[0], "test scanner started") + self.assertTrue(any("test scanner still running" in message for message in messages)) + self.assertTrue(any("test scanner complete" in message for message in messages)) + def test_license_files_are_split_before_scancode_and_paths_are_collapsed(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -62,12 +93,13 @@ def test_license_files_are_split_before_scancode_and_paths_are_collapsed(self): temporary.mkdir() commands = [] - def fake_scanner(command, **_kwargs): + def fake_csplit(command, **_kwargs): commands.append(command) - if command[0] == "csplit": - prefix = Path(command[command.index("-f") + 1]) - prefix.with_name("license-00").write_text("license text") - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + prefix = Path(command[command.index("-f") + 1]) + prefix.with_name("license-00").write_text("license text") + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def fake_scancode(command, **_kwargs): output = Path(command[command.index("--json") + 1]) output.write_text(json.dumps({ "files": [{ @@ -79,10 +111,18 @@ def fake_scanner(command, **_kwargs): }], }], })) - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + class FakeProcess: + returncode = 0 + + def communicate(self, timeout=None): + return "", "" + + return FakeProcess() with patch("generator.shutil.which", return_value="scancode"), patch( - "generator.subprocess.run", side_effect=fake_scanner + "generator.subprocess.run", side_effect=fake_csplit + ), patch( + "generator.subprocess.Popen", side_effect=fake_scancode ): report = scan_licenses(root, temporary) From 4f60d028afe349a18e336724a96c699ea2264743 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:52:30 +0000 Subject: [PATCH 08/24] Preserve SBOM composer executable mode Keep the existing executable bit when synchronizing the signed agent-branch update. --- sbom-generator/compose.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sbom-generator/compose.py diff --git a/sbom-generator/compose.py b/sbom-generator/compose.py old mode 100644 new mode 100755 From 22a8bfe8f23f89dbe82b8135d28129b2dcbf40cd Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:59:16 +0000 Subject: [PATCH 09/24] Identify SBOM generator builds by Git revision --- .github/workflows/sbom-generator.yml | 1 + sbom-generator/Dockerfile | 4 ++++ sbom-generator/README.md | 10 ++++++++-- sbom-generator/compose.py | 7 ++++--- sbom-generator/tests/test_compose.py | 9 ++++++--- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sbom-generator.yml b/.github/workflows/sbom-generator.yml index d4ec03b3..66ba1fe1 100644 --- a/.github/workflows/sbom-generator.yml +++ b/.github/workflows/sbom-generator.yml @@ -65,6 +65,7 @@ jobs: if [[ "$EVENT_NAME" == push || "$PUBLISH" == true ]]; then options+=(--push); fi docker buildx build \ --platform linux/amd64,linux/arm64 \ + --build-arg "SBOM_GENERATOR_REVISION=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "$image:latest" \ --tag "$image:sha-${GITHUB_SHA}" \ diff --git a/sbom-generator/Dockerfile b/sbom-generator/Dockerfile index ec7ab001..98913dae 100644 --- a/sbom-generator/Dockerfile +++ b/sbom-generator/Dockerfile @@ -24,5 +24,9 @@ RUN apt-get update \ WORKDIR /opt/cnpg-sbom-generator COPY compose.py generator.py hooks.py ./ +ARG SBOM_GENERATOR_REVISION=unknown +ENV SBOM_GENERATOR_REVISION=${SBOM_GENERATOR_REVISION} +LABEL org.opencontainers.image.revision=${SBOM_GENERATOR_REVISION} + ENV PYTHONUNBUFFERED=1 ENTRYPOINT ["python3", "/opt/cnpg-sbom-generator/generator.py"] diff --git a/sbom-generator/README.md b/sbom-generator/README.md index 3f1b78e9..9941c2b7 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -150,9 +150,15 @@ image index. A multi-platform index therefore contains one image and one combined provenance/SPDX attestation for each platform. The SPDX creation metadata identifies the generator as -`Tool: cnpg-sbom-generator-1`. Its document annotation retains the generator -name, version, and source repository URL; the immutable generator image digest +`Tool: cnpg-sbom-generator-`. The publishing workflow embeds its full +build Git SHA in the generator image through `SBOM_GENERATOR_REVISION`; this +also sets the image's `org.opencontainers.image.revision` label. Its document +annotation retains the generator name, Git SHA as `generatorVersion`, and source +repository URL; the immutable generator image digest selected by the workflow provides the stronger reproducibility boundary. +Local image builds can pass `--build-arg SBOM_GENERATOR_REVISION=$(git rev-parse HEAD)` +when building from an unchanged checkout. Without a supplied revision, the +version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. Long Syft and ScanCode subprocesses emit a heartbeat every 30 seconds, and diff --git a/sbom-generator/compose.py b/sbom-generator/compose.py index dae0c346..6d9b39d5 100755 --- a/sbom-generator/compose.py +++ b/sbom-generator/compose.py @@ -11,6 +11,7 @@ import argparse import hashlib import json +import os import re import sys from collections import defaultdict @@ -22,7 +23,6 @@ EXTENSION_PACKAGE_ID = "SPDXRef-Package-extension-payload" GENERATOR_NAME = "cnpg-sbom-generator" -GENERATOR_VERSION = "1" GENERATOR_REPOSITORY = "https://github.com/cnpg-extensions/postgres-extensions-containers" LICENSE_REF = re.compile(r"LicenseRef-[A-Za-z0-9][A-Za-z0-9.-]*") LICENSE_OPERATOR = re.compile(r"\s+(?:AND|OR|WITH)\s+") @@ -503,13 +503,14 @@ def add_synthetic_file(record: dict[str, str], owner: str | None = None) -> None raise ValueError(f"unsupported target platform: {platform!r}") creation_info = output.setdefault("creationInfo", {}) creators = list(creation_info.get("creators", [])) - generator_creator = f"Tool: {GENERATOR_NAME}-{GENERATOR_VERSION}" + generator_version = os.getenv("SBOM_GENERATOR_REVISION") or "unknown" + generator_creator = f"Tool: {GENERATOR_NAME}-{generator_version}" if generator_creator not in creators: creators.append(generator_creator) creation_info["creators"] = creators metadata = { "generator": GENERATOR_NAME, - "generatorVersion": GENERATOR_VERSION, + "generatorVersion": generator_version, "generatorRepository": GENERATOR_REPOSITORY, "platform": platform, "evidence": evidence or {}, diff --git a/sbom-generator/tests/test_compose.py b/sbom-generator/tests/test_compose.py index f56e2878..94e6362f 100644 --- a/sbom-generator/tests/test_compose.py +++ b/sbom-generator/tests/test_compose.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).parents[1])) @@ -88,7 +89,9 @@ def test_composes_only_shipped_files_and_owned_packages(self): self.assertNotIn("build-only", json.dumps(output)) self.assertFalse("subject" in output) + @patch.dict(os.environ, {"SBOM_GENERATOR_REVISION": "abc123" * 6 + "abcd"}) def test_generator_metadata_identifies_version_and_repository(self): + revision = os.environ["SBOM_GENERATOR_REVISION"] output = compose( builder_document(), extension_name="plr", @@ -97,12 +100,12 @@ def test_generator_metadata_identifies_version_and_repository(self): ) generator_annotation = next( annotation for annotation in output["annotations"] - if annotation["annotator"] == "Tool: cnpg-sbom-generator-1" + if annotation["annotator"] == f"Tool: cnpg-sbom-generator-{revision}" ) metadata = json.loads(generator_annotation["comment"]) - self.assertIn("Tool: cnpg-sbom-generator-1", output["creationInfo"]["creators"]) + self.assertIn(f"Tool: cnpg-sbom-generator-{revision}", output["creationInfo"]["creators"]) self.assertEqual(metadata["generator"], "cnpg-sbom-generator") - self.assertEqual(metadata["generatorVersion"], "1") + self.assertEqual(metadata["generatorVersion"], revision) self.assertEqual( metadata["generatorRepository"], "https://github.com/cnpg-extensions/postgres-extensions-containers", From 0983fb7e7148050b8989951e97a0e0bcb1cdf086 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:20:10 +0000 Subject: [PATCH 10/24] Update pinned SBOM generator image digest --- .github/workflows/bake_targets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 20316932..9735abc6 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -61,7 +61,7 @@ jobs: registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator - sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:cbd49cfd14fd66e7ac3c3c25feea70f9ad7f62a3b4faf1c7cbea1e21602ed9bb + sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:37088112d4eff911a69d5bb1dbd17b40fa5b87bad31e7b05da569ce47440944e with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . From 8217f71a42bb1cf2b7714e7b2c365e8d78854db4 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:33:07 +0000 Subject: [PATCH 11/24] Clarify ScanCode total license chunk count --- sbom-generator/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 47855a3b..743d76aa 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -220,7 +220,7 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: scan_root = prepare_license_scan_root(final_root, temporary) output = temporary / "scancode.json" try: - input_files = sum(1 for path in scan_root.rglob("*") if path.is_file()) + license_chunks = sum(1 for path in scan_root.rglob("*") if path.is_file()) run_command_with_progress( [ scancode, @@ -230,7 +230,7 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: str(output), str(scan_root), ], - f"ScanCode license scan ({input_files} input files)", + f"ScanCode license scan ({license_chunks} license chunks to scan in total)", ) except FileNotFoundError as error: raise RuntimeError("scancode is required when the final payload has /licenses") from error From dbc95175d29b63f753e877bf9983c046ee71d85c Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:35:57 +0000 Subject: [PATCH 12/24] Report completed ScanCode license chunks --- sbom-generator/README.md | 6 ++- sbom-generator/generator.py | 51 ++++++++++++++++++++------ sbom-generator/tests/test_generator.py | 28 +++++++++++++- 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index 9941c2b7..b7bbe77e 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -161,7 +161,11 @@ when building from an unchanged checkout. Without a supplied revision, the version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. -Long Syft and ScanCode subprocesses emit a heartbeat every 30 seconds, and +Long Syft subprocesses emit a heartbeat every 30 seconds. ScanCode reports +completed work as `ScanCode: 240 of 1,200 license chunks scanned`, at most once +every 30 seconds while work advances, plus the initial and final counts. The +count comes from ScanCode's per-file completion events; scanner errors still +fail the build even if all chunks were processed. Also, license-file preparation reports its file and chunk counts. The examples below use the H3 image produced by this repository. Replace diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 743d76aa..018032c1 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -20,6 +20,7 @@ import sys import tempfile import time +from collections import deque from pathlib import Path from typing import Any, Sequence @@ -126,7 +127,7 @@ def progress(message: str) -> None: def run_command_with_progress( - command: Sequence[str], label: str + command: Sequence[str], label: str, *, license_chunks: int | None = None ) -> subprocess.CompletedProcess: """Run a scanner while keeping long-running phases visible in BuildKit logs.""" @@ -136,21 +137,47 @@ def run_command_with_progress( process = subprocess.Popen( list(command), stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT if license_chunks is not None else subprocess.PIPE, text=True, ) except FileNotFoundError: raise - while True: - try: - stdout, stderr = process.communicate(timeout=PROGRESS_INTERVAL_SECONDS) - break - except subprocess.TimeoutExpired: - progress( - f"{label} still running " - f"({time.monotonic() - started:.0f}s elapsed)" - ) + if license_chunks is not None: + # ScanCode --verbose emits Scanned: only after a file scan returns. + # Suppress individual paths, retaining a bounded diagnostic tail. + completed = set() + tail = deque(maxlen=100) + last_report = started + last_count = 0 + progress(f"ScanCode: 0 of {license_chunks:,} license chunks scanned") + for line in process.stdout: + tail.append(line) + if line.startswith("Scanned: "): + completed.add(line.removeprefix("Scanned: ").strip()) + count = len(completed) + now = time.monotonic() + if count != last_count and ( + now - last_report >= PROGRESS_INTERVAL_SECONDS + or count == license_chunks + ): + progress(f"ScanCode: {count:,} of {license_chunks:,} license chunks scanned") + last_report, last_count = now, count + process.stdout.close() + process.wait() + if len(completed) != last_count: + progress(f"ScanCode: {len(completed):,} of {license_chunks:,} license chunks scanned") + stdout, stderr = "".join(tail), "" + else: + while True: + try: + stdout, stderr = process.communicate(timeout=PROGRESS_INTERVAL_SECONDS) + break + except subprocess.TimeoutExpired: + progress( + f"{label} still running " + f"({time.monotonic() - started:.0f}s elapsed)" + ) result = subprocess.CompletedProcess( list(command), process.returncode, stdout, stderr @@ -224,6 +251,7 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: run_command_with_progress( [ scancode, + "--verbose", "--license", "--license-references", "--json", @@ -231,6 +259,7 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: str(scan_root), ], f"ScanCode license scan ({license_chunks} license chunks to scan in total)", + license_chunks=license_chunks, ) except FileNotFoundError as error: raise RuntimeError("scancode is required when the final payload has /licenses") from error diff --git a/sbom-generator/tests/test_generator.py b/sbom-generator/tests/test_generator.py index d29903d5..42fc47c2 100644 --- a/sbom-generator/tests/test_generator.py +++ b/sbom-generator/tests/test_generator.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import hashlib +import io import json import os import subprocess @@ -113,9 +114,10 @@ def fake_scancode(command, **_kwargs): })) class FakeProcess: returncode = 0 + stdout = io.StringIO("Scanned: licenses/copyright/license-00\n") - def communicate(self, timeout=None): - return "", "" + def wait(self): + return 0 return FakeProcess() @@ -135,6 +137,28 @@ def communicate(self, timeout=None): self.assertIn("/^License:/", split_command) self.assertIn("{*}", split_command) + def test_scancode_counts_unique_completion_events_and_preserves_failure(self): + for exit_code in (0, 1): + with self.subTest(exit_code=exit_code): + messages = [] + command = [sys.executable, "-c", ( + "import sys; " + "print('Setup plugins...', file=sys.stderr); " + "print('Scanned: /licenses/a', file=sys.stderr); " + "print('Scanned: /licenses/a', file=sys.stderr); " + "print('Scanned: /licenses/b', file=sys.stderr); " + "print('scan diagnostic', file=sys.stderr); " + f"sys.exit({exit_code})" + )] + with patch("generator.progress", side_effect=messages.append): + if exit_code: + with self.assertRaisesRegex(RuntimeError, "scan diagnostic"): + run_command_with_progress(command, "ScanCode", license_chunks=1200) + else: + run_command_with_progress(command, "ScanCode", license_chunks=1200) + self.assertIn("ScanCode: 2 of 1,200 license chunks scanned", messages) + self.assertFalse(any("still running" in message for message in messages)) + if __name__ == "__main__": unittest.main() From 0dcd8495590945a5dbcb8f6b930259532040bf04 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:39:30 +0000 Subject: [PATCH 13/24] Report ScanCode progress every hundred chunks --- sbom-generator/README.md | 4 ++-- sbom-generator/generator.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index b7bbe77e..ba92a98e 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -162,8 +162,8 @@ version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. Long Syft subprocesses emit a heartbeat every 30 seconds. ScanCode reports -completed work as `ScanCode: 240 of 1,200 license chunks scanned`, at most once -every 30 seconds while work advances, plus the initial and final counts. The +completed work as `ScanCode: 200 of 1,200 license chunks scanned`, every +100 completed chunks, plus the initial and final counts. The count comes from ScanCode's per-file completion events; scanner errors still fail the build even if all chunks were processed. Also, license-file preparation reports its file and chunk counts. diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 018032c1..8478b898 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -148,7 +148,6 @@ def run_command_with_progress( # Suppress individual paths, retaining a bounded diagnostic tail. completed = set() tail = deque(maxlen=100) - last_report = started last_count = 0 progress(f"ScanCode: 0 of {license_chunks:,} license chunks scanned") for line in process.stdout: @@ -156,13 +155,12 @@ def run_command_with_progress( if line.startswith("Scanned: "): completed.add(line.removeprefix("Scanned: ").strip()) count = len(completed) - now = time.monotonic() if count != last_count and ( - now - last_report >= PROGRESS_INTERVAL_SECONDS + count % 100 == 0 or count == license_chunks ): progress(f"ScanCode: {count:,} of {license_chunks:,} license chunks scanned") - last_report, last_count = now, count + last_count = count process.stdout.close() process.wait() if len(completed) != last_count: From 13c3a266e523b25f1d4523046fb30071e671b157 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:41:44 +0000 Subject: [PATCH 14/24] Revert "Report ScanCode progress every hundred chunks" This reverts commit 10d9f5c1fc3d0164ff0f36730ccd50a5f23f4e74. --- sbom-generator/README.md | 4 ++-- sbom-generator/generator.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index ba92a98e..b7bbe77e 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -162,8 +162,8 @@ version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. Long Syft subprocesses emit a heartbeat every 30 seconds. ScanCode reports -completed work as `ScanCode: 200 of 1,200 license chunks scanned`, every -100 completed chunks, plus the initial and final counts. The +completed work as `ScanCode: 240 of 1,200 license chunks scanned`, at most once +every 30 seconds while work advances, plus the initial and final counts. The count comes from ScanCode's per-file completion events; scanner errors still fail the build even if all chunks were processed. Also, license-file preparation reports its file and chunk counts. diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 8478b898..018032c1 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -148,6 +148,7 @@ def run_command_with_progress( # Suppress individual paths, retaining a bounded diagnostic tail. completed = set() tail = deque(maxlen=100) + last_report = started last_count = 0 progress(f"ScanCode: 0 of {license_chunks:,} license chunks scanned") for line in process.stdout: @@ -155,12 +156,13 @@ def run_command_with_progress( if line.startswith("Scanned: "): completed.add(line.removeprefix("Scanned: ").strip()) count = len(completed) + now = time.monotonic() if count != last_count and ( - count % 100 == 0 + now - last_report >= PROGRESS_INTERVAL_SECONDS or count == license_chunks ): progress(f"ScanCode: {count:,} of {license_chunks:,} license chunks scanned") - last_count = count + last_report, last_count = now, count process.stdout.close() process.wait() if len(completed) != last_count: From ee2a5ccc91a38fbb745f2dd44e2c0ae3d360dc44 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:42:55 +0000 Subject: [PATCH 15/24] Report ScanCode progress every ten seconds --- sbom-generator/README.md | 2 +- sbom-generator/generator.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index b7bbe77e..bb622e0f 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -163,7 +163,7 @@ version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. Long Syft subprocesses emit a heartbeat every 30 seconds. ScanCode reports completed work as `ScanCode: 240 of 1,200 license chunks scanned`, at most once -every 30 seconds while work advances, plus the initial and final counts. The +every 10 seconds while work advances, plus the initial and final counts. The count comes from ScanCode's per-file completion events; scanner errors still fail the build even if all chunks were processed. Also, license-file preparation reports its file and chunk counts. diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 018032c1..035acc4b 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -38,6 +38,7 @@ INTOTO_STATEMENT_TYPE = "https://in-toto.io/Statement/v1" SPDX_PREDICATE_TYPE = "https://spdx.dev/Document" PROGRESS_INTERVAL_SECONDS = 30 +SCANCODE_PROGRESS_INTERVAL_SECONDS = 10 def require_directory(value: str | None, variable: str) -> Path: @@ -158,7 +159,7 @@ def run_command_with_progress( count = len(completed) now = time.monotonic() if count != last_count and ( - now - last_report >= PROGRESS_INTERVAL_SECONDS + now - last_report >= SCANCODE_PROGRESS_INTERVAL_SECONDS or count == license_chunks ): progress(f"ScanCode: {count:,} of {license_chunks:,} license chunks scanned") From b9965f497e0d0b12eef0b72facee5f72c82f9041 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:45:19 +0000 Subject: [PATCH 16/24] Use one ten-second scanner progress interval --- sbom-generator/README.md | 2 +- sbom-generator/generator.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index bb622e0f..9548c652 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -161,7 +161,7 @@ when building from an unchanged checkout. Without a supplied revision, the version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. -Long Syft subprocesses emit a heartbeat every 30 seconds. ScanCode reports +Long Syft subprocesses emit a heartbeat every 10 seconds. ScanCode reports completed work as `ScanCode: 240 of 1,200 license chunks scanned`, at most once every 10 seconds while work advances, plus the initial and final counts. The count comes from ScanCode's per-file completion events; scanner errors still diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 035acc4b..76fd58c9 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -37,8 +37,7 @@ INTOTO_STATEMENT_TYPE = "https://in-toto.io/Statement/v1" SPDX_PREDICATE_TYPE = "https://spdx.dev/Document" -PROGRESS_INTERVAL_SECONDS = 30 -SCANCODE_PROGRESS_INTERVAL_SECONDS = 10 +PROGRESS_INTERVAL_SECONDS = 10 def require_directory(value: str | None, variable: str) -> Path: @@ -159,7 +158,7 @@ def run_command_with_progress( count = len(completed) now = time.monotonic() if count != last_count and ( - now - last_report >= SCANCODE_PROGRESS_INTERVAL_SECONDS + now - last_report >= PROGRESS_INTERVAL_SECONDS or count == license_chunks ): progress(f"ScanCode: {count:,} of {license_chunks:,} license chunks scanned") From 553a6e9bc1d672e75ba5b455d451cff5349468e6 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:50:27 +0000 Subject: [PATCH 17/24] Serialize BuildKit operations to reduce SBOM memory pressure --- .github/workflows/bake_targets.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 9735abc6..0cdc8029 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -39,12 +39,17 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 with: platforms: 'linux/arm64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 + with: + # Run SBOM generators sequentially to reduce peak runner memory use. + buildkitd-config-inline: | + [worker.oci] + max-parallelism = 1 - name: Expose builder stage to SBOM generator env: From 7cf86cc9b6b45cc0d8adb64d057fc0a47ac8f86c Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:00:09 +0000 Subject: [PATCH 18/24] Update pinned SBOM generator to latest image --- .github/workflows/bake_targets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 0cdc8029..bf1ee1b4 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -66,7 +66,7 @@ jobs: registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator - sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:37088112d4eff911a69d5bb1dbd17b40fa5b87bad31e7b05da569ce47440944e + sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:c23d06ef1d48074c56fa473a134aa887b06be7e5d3ab43c1b06cce2f899eb29e with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . From 478fcc1a04caa89c39e3b5460e0d75f3ed259d5b Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:20:32 +0000 Subject: [PATCH 19/24] Simplify ScanCode logging to per-run timing --- sbom-generator/README.md | 10 ++-- sbom-generator/generator.py | 65 +++++++++----------------- sbom-generator/tests/test_generator.py | 36 ++------------ 3 files changed, 29 insertions(+), 82 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index 9548c652..5cb97a2d 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -161,12 +161,10 @@ when building from an unchanged checkout. Without a supplied revision, the version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. -Long Syft subprocesses emit a heartbeat every 10 seconds. ScanCode reports -completed work as `ScanCode: 240 of 1,200 license chunks scanned`, at most once -every 10 seconds while work advances, plus the initial and final counts. The -count comes from ScanCode's per-file completion events; scanner errors still -fail the build even if all chunks were processed. Also, -license-file preparation reports its file and chunk counts. +Long Syft subprocesses emit a heartbeat every 10 seconds. Each ScanCode invocation +logs its start and a completion message with the chunk count and elapsed time. +Scanner diagnostics are included on failure; individual chunks are not logged. +License-file preparation reports its file and chunk counts. The examples below use the H3 image produced by this repository. Replace `INDEX_DIGEST` with the immutable index digest returned by diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 76fd58c9..15469bd2 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -20,7 +20,6 @@ import sys import tempfile import time -from collections import deque from pathlib import Path from typing import Any, Sequence @@ -127,7 +126,7 @@ def progress(message: str) -> None: def run_command_with_progress( - command: Sequence[str], label: str, *, license_chunks: int | None = None + command: Sequence[str], label: str ) -> subprocess.CompletedProcess: """Run a scanner while keeping long-running phases visible in BuildKit logs.""" @@ -137,47 +136,21 @@ def run_command_with_progress( process = subprocess.Popen( list(command), stdout=subprocess.PIPE, - stderr=subprocess.STDOUT if license_chunks is not None else subprocess.PIPE, + stderr=subprocess.PIPE, text=True, ) except FileNotFoundError: raise - if license_chunks is not None: - # ScanCode --verbose emits Scanned: only after a file scan returns. - # Suppress individual paths, retaining a bounded diagnostic tail. - completed = set() - tail = deque(maxlen=100) - last_report = started - last_count = 0 - progress(f"ScanCode: 0 of {license_chunks:,} license chunks scanned") - for line in process.stdout: - tail.append(line) - if line.startswith("Scanned: "): - completed.add(line.removeprefix("Scanned: ").strip()) - count = len(completed) - now = time.monotonic() - if count != last_count and ( - now - last_report >= PROGRESS_INTERVAL_SECONDS - or count == license_chunks - ): - progress(f"ScanCode: {count:,} of {license_chunks:,} license chunks scanned") - last_report, last_count = now, count - process.stdout.close() - process.wait() - if len(completed) != last_count: - progress(f"ScanCode: {len(completed):,} of {license_chunks:,} license chunks scanned") - stdout, stderr = "".join(tail), "" - else: - while True: - try: - stdout, stderr = process.communicate(timeout=PROGRESS_INTERVAL_SECONDS) - break - except subprocess.TimeoutExpired: - progress( - f"{label} still running " - f"({time.monotonic() - started:.0f}s elapsed)" - ) + while True: + try: + stdout, stderr = process.communicate(timeout=PROGRESS_INTERVAL_SECONDS) + break + except subprocess.TimeoutExpired: + progress( + f"{label} still running " + f"({time.monotonic() - started:.0f}s elapsed)" + ) result = subprocess.CompletedProcess( list(command), process.returncode, stdout, stderr @@ -246,23 +219,29 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: raise RuntimeError("scancode is required when the final payload has /licenses") scan_root = prepare_license_scan_root(final_root, temporary) output = temporary / "scancode.json" + license_chunks = sum(1 for path in scan_root.rglob("*") if path.is_file()) + progress(f"ScanCode license scan ({license_chunks:,} license chunks to scan in total) started") + started = time.monotonic() try: - license_chunks = sum(1 for path in scan_root.rglob("*") if path.is_file()) - run_command_with_progress( + subprocess.run( [ scancode, - "--verbose", "--license", "--license-references", "--json", str(output), str(scan_root), ], - f"ScanCode license scan ({license_chunks} license chunks to scan in total)", - license_chunks=license_chunks, + check=True, + capture_output=True, + text=True, ) except FileNotFoundError as error: raise RuntimeError("scancode is required when the final payload has /licenses") from error + except subprocess.CalledProcessError as error: + detail = (error.stderr or error.stdout or "scanner failed").strip() + raise RuntimeError(f"ScanCode failed after {time.monotonic() - started:.1f}s: {detail}") from error + progress(f"ScanCode processed {license_chunks:,} license chunks in {time.monotonic() - started:.1f}s") try: with output.open(encoding="utf-8") as stream: report = json.load(stream) diff --git a/sbom-generator/tests/test_generator.py b/sbom-generator/tests/test_generator.py index 42fc47c2..96596d70 100644 --- a/sbom-generator/tests/test_generator.py +++ b/sbom-generator/tests/test_generator.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import hashlib -import io import json import os import subprocess @@ -95,6 +94,8 @@ def test_license_files_are_split_before_scancode_and_paths_are_collapsed(self): commands = [] def fake_csplit(command, **_kwargs): + if command[0] == "scancode": + return fake_scancode(command, **_kwargs) commands.append(command) prefix = Path(command[command.index("-f") + 1]) prefix.with_name("license-00").write_text("license text") @@ -112,19 +113,10 @@ def fake_scancode(command, **_kwargs): }], }], })) - class FakeProcess: - returncode = 0 - stdout = io.StringIO("Scanned: licenses/copyright/license-00\n") - - def wait(self): - return 0 - - return FakeProcess() + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") with patch("generator.shutil.which", return_value="scancode"), patch( "generator.subprocess.run", side_effect=fake_csplit - ), patch( - "generator.subprocess.Popen", side_effect=fake_scancode ): report = scan_licenses(root, temporary) @@ -137,28 +129,6 @@ def wait(self): self.assertIn("/^License:/", split_command) self.assertIn("{*}", split_command) - def test_scancode_counts_unique_completion_events_and_preserves_failure(self): - for exit_code in (0, 1): - with self.subTest(exit_code=exit_code): - messages = [] - command = [sys.executable, "-c", ( - "import sys; " - "print('Setup plugins...', file=sys.stderr); " - "print('Scanned: /licenses/a', file=sys.stderr); " - "print('Scanned: /licenses/a', file=sys.stderr); " - "print('Scanned: /licenses/b', file=sys.stderr); " - "print('scan diagnostic', file=sys.stderr); " - f"sys.exit({exit_code})" - )] - with patch("generator.progress", side_effect=messages.append): - if exit_code: - with self.assertRaisesRegex(RuntimeError, "scan diagnostic"): - run_command_with_progress(command, "ScanCode", license_chunks=1200) - else: - run_command_with_progress(command, "ScanCode", license_chunks=1200) - self.assertIn("ScanCode: 2 of 1,200 license chunks scanned", messages) - self.assertFalse(any("still running" in message for message in messages)) - if __name__ == "__main__": unittest.main() From ff514a71b553a99b4ef1f7ace6158a979211ae46 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:22:18 +0000 Subject: [PATCH 20/24] Log each completed ScanCode license chunk --- sbom-generator/README.md | 3 ++- sbom-generator/generator.py | 24 +++++++++++++++++------- sbom-generator/tests/test_generator.py | 21 +++++++++++++++------ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index 5cb97a2d..24e09abe 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -163,7 +163,8 @@ version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. Long Syft subprocesses emit a heartbeat every 10 seconds. Each ScanCode invocation logs its start and a completion message with the chunk count and elapsed time. -Scanner diagnostics are included on failure; individual chunks are not logged. +Each completed chunk is logged as `ScanCode processed file /path/to/license-00`. +ScanCode's other output and diagnostics are streamed to the build log. License-file preparation reports its file and chunk counts. The examples below use the H3 image produced by this repository. Replace diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 15469bd2..0eb87ebe 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -223,24 +223,34 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: progress(f"ScanCode license scan ({license_chunks:,} license chunks to scan in total) started") started = time.monotonic() try: - subprocess.run( + with subprocess.Popen( [ scancode, + "--verbose", "--license", "--license-references", "--json", str(output), str(scan_root), ], - check=True, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, - ) + ) as process: + for line in process.stdout: + # ScanCode styles its completion prefix with ANSI escapes. + plain = re.sub(r"\x1b\[[0-9;]*m", "", line).strip() + if plain.startswith("Scanned: "): + print(f"ScanCode processed file {plain.removeprefix('Scanned: ')}", flush=True) + else: + print(line, end="", flush=True) + if process.wait(): + raise RuntimeError( + f"ScanCode failed after {time.monotonic() - started:.1f}s " + f"with exit code {process.returncode}; see scanner output above" + ) except FileNotFoundError as error: raise RuntimeError("scancode is required when the final payload has /licenses") from error - except subprocess.CalledProcessError as error: - detail = (error.stderr or error.stdout or "scanner failed").strip() - raise RuntimeError(f"ScanCode failed after {time.monotonic() - started:.1f}s: {detail}") from error progress(f"ScanCode processed {license_chunks:,} license chunks in {time.monotonic() - started:.1f}s") try: with output.open(encoding="utf-8") as stream: diff --git a/sbom-generator/tests/test_generator.py b/sbom-generator/tests/test_generator.py index 96596d70..43a59726 100644 --- a/sbom-generator/tests/test_generator.py +++ b/sbom-generator/tests/test_generator.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import hashlib +import io +from contextlib import redirect_stdout import json import os import subprocess @@ -7,7 +9,7 @@ import tempfile import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch sys.path.insert(0, str(Path(__file__).parents[1])) @@ -94,14 +96,13 @@ def test_license_files_are_split_before_scancode_and_paths_are_collapsed(self): commands = [] def fake_csplit(command, **_kwargs): - if command[0] == "scancode": - return fake_scancode(command, **_kwargs) commands.append(command) prefix = Path(command[command.index("-f") + 1]) prefix.with_name("license-00").write_text("license text") return subprocess.CompletedProcess(command, 0, stdout="", stderr="") def fake_scancode(command, **_kwargs): + self.assertIn("--verbose", command) output = Path(command[command.index("--json") + 1]) output.write_text(json.dumps({ "files": [{ @@ -113,12 +114,20 @@ def fake_scancode(command, **_kwargs): }], }], })) - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - + process = MagicMock() + process.__enter__.return_value = process + process.stdout = io.StringIO( + "\x1b[0mScanned: \x1b[0m\x1b[32m/licenses/copyright/license-00\x1b[0m\n" + ) + process.wait.return_value = 0 + return process + + messages = io.StringIO() with patch("generator.shutil.which", return_value="scancode"), patch( "generator.subprocess.run", side_effect=fake_csplit - ): + ), patch("generator.subprocess.Popen", side_effect=fake_scancode), redirect_stdout(messages): report = scan_licenses(root, temporary) + self.assertEqual(messages.getvalue(), "ScanCode processed file /licenses/copyright/license-00\n") self.assertEqual(report["files"][0]["path"], "licenses/copyright") self.assertEqual( From 4276672436bf07e580aa8c6f9275fd3b08ffd945 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:42:35 +0000 Subject: [PATCH 21/24] Pass ScanCode output directly to build logs --- sbom-generator/README.md | 4 ++-- sbom-generator/generator.py | 21 +++++---------------- sbom-generator/tests/test_generator.py | 23 +++++++++-------------- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index 24e09abe..7a9c8446 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -163,8 +163,8 @@ version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. Long Syft subprocesses emit a heartbeat every 10 seconds. Each ScanCode invocation logs its start and a completion message with the chunk count and elapsed time. -Each completed chunk is logged as `ScanCode processed file /path/to/license-00`. -ScanCode's other output and diagnostics are streamed to the build log. +ScanCode's output and diagnostics go directly to stdout without filtering. +ScanCode controls its own progress display; non-terminal logs may omit per-file messages. License-file preparation reports its file and chunk counts. The examples below use the H3 image produced by this repository. Replace diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 0eb87ebe..7cdbf041 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -223,7 +223,7 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: progress(f"ScanCode license scan ({license_chunks:,} license chunks to scan in total) started") started = time.monotonic() try: - with subprocess.Popen( + subprocess.run( [ scancode, "--verbose", @@ -233,24 +233,13 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: str(output), str(scan_root), ], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, - ) as process: - for line in process.stdout: - # ScanCode styles its completion prefix with ANSI escapes. - plain = re.sub(r"\x1b\[[0-9;]*m", "", line).strip() - if plain.startswith("Scanned: "): - print(f"ScanCode processed file {plain.removeprefix('Scanned: ')}", flush=True) - else: - print(line, end="", flush=True) - if process.wait(): - raise RuntimeError( - f"ScanCode failed after {time.monotonic() - started:.1f}s " - f"with exit code {process.returncode}; see scanner output above" - ) + check=True, + ) except FileNotFoundError as error: raise RuntimeError("scancode is required when the final payload has /licenses") from error + except subprocess.CalledProcessError as error: + raise RuntimeError(f"ScanCode failed with exit code {error.returncode}; see output above") from error progress(f"ScanCode processed {license_chunks:,} license chunks in {time.monotonic() - started:.1f}s") try: with output.open(encoding="utf-8") as stream: diff --git a/sbom-generator/tests/test_generator.py b/sbom-generator/tests/test_generator.py index 43a59726..1205b2e2 100644 --- a/sbom-generator/tests/test_generator.py +++ b/sbom-generator/tests/test_generator.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 import hashlib -import io -from contextlib import redirect_stdout import json import os import subprocess @@ -9,7 +7,7 @@ import tempfile import unittest from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).parents[1])) @@ -96,6 +94,8 @@ def test_license_files_are_split_before_scancode_and_paths_are_collapsed(self): commands = [] def fake_csplit(command, **_kwargs): + if command[0] == "scancode": + return fake_scancode(command, **_kwargs) commands.append(command) prefix = Path(command[command.index("-f") + 1]) prefix.with_name("license-00").write_text("license text") @@ -103,6 +103,9 @@ def fake_csplit(command, **_kwargs): def fake_scancode(command, **_kwargs): self.assertIn("--verbose", command) + self.assertNotIn("stdout", _kwargs) + self.assertEqual(_kwargs["stderr"], subprocess.STDOUT) + self.assertTrue(_kwargs["check"]) output = Path(command[command.index("--json") + 1]) output.write_text(json.dumps({ "files": [{ @@ -114,20 +117,12 @@ def fake_scancode(command, **_kwargs): }], }], })) - process = MagicMock() - process.__enter__.return_value = process - process.stdout = io.StringIO( - "\x1b[0mScanned: \x1b[0m\x1b[32m/licenses/copyright/license-00\x1b[0m\n" - ) - process.wait.return_value = 0 - return process - - messages = io.StringIO() + return subprocess.CompletedProcess(command, 0) + with patch("generator.shutil.which", return_value="scancode"), patch( "generator.subprocess.run", side_effect=fake_csplit - ), patch("generator.subprocess.Popen", side_effect=fake_scancode), redirect_stdout(messages): + ): report = scan_licenses(root, temporary) - self.assertEqual(messages.getvalue(), "ScanCode processed file /licenses/copyright/license-00\n") self.assertEqual(report["files"][0]["path"], "licenses/copyright") self.assertEqual( From 101874a68c54c233f013f8fb2a624d55be1e3166 Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:57:04 +0000 Subject: [PATCH 22/24] Remove redundant ScanCode wrapper status messages --- sbom-generator/README.md | 3 +-- sbom-generator/generator.py | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/sbom-generator/README.md b/sbom-generator/README.md index 7a9c8446..4658b308 100644 --- a/sbom-generator/README.md +++ b/sbom-generator/README.md @@ -161,8 +161,7 @@ when building from an unchanged checkout. Without a supplied revision, the version is explicitly `unknown`, including when running the composer directly. During generation, BuildKit logs show phase start/completion and elapsed time. -Long Syft subprocesses emit a heartbeat every 10 seconds. Each ScanCode invocation -logs its start and a completion message with the chunk count and elapsed time. +Long Syft subprocesses emit a heartbeat every 10 seconds. ScanCode's output and diagnostics go directly to stdout without filtering. ScanCode controls its own progress display; non-terminal logs may omit per-file messages. License-file preparation reports its file and chunk counts. diff --git a/sbom-generator/generator.py b/sbom-generator/generator.py index 7cdbf041..06a3f151 100644 --- a/sbom-generator/generator.py +++ b/sbom-generator/generator.py @@ -212,16 +212,12 @@ def scan_builder(builder: Path, temporary: Path) -> dict[str, Any]: def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: licenses = final_root / "licenses" if not licenses.exists(): - progress("no /licenses directory; skipping ScanCode") return {"files": []} scancode = shutil.which("scancode") if not scancode: raise RuntimeError("scancode is required when the final payload has /licenses") scan_root = prepare_license_scan_root(final_root, temporary) output = temporary / "scancode.json" - license_chunks = sum(1 for path in scan_root.rglob("*") if path.is_file()) - progress(f"ScanCode license scan ({license_chunks:,} license chunks to scan in total) started") - started = time.monotonic() try: subprocess.run( [ @@ -240,7 +236,6 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: raise RuntimeError("scancode is required when the final payload has /licenses") from error except subprocess.CalledProcessError as error: raise RuntimeError(f"ScanCode failed with exit code {error.returncode}; see output above") from error - progress(f"ScanCode processed {license_chunks:,} license chunks in {time.monotonic() - started:.1f}s") try: with output.open(encoding="utf-8") as stream: report = json.load(stream) @@ -249,7 +244,6 @@ def scan_licenses(final_root: Path, temporary: Path) -> dict[str, Any]: if not isinstance(report, dict): raise RuntimeError("scancode output is not a JSON object") normalize_scancode_report_paths(report, scan_root, final_root) - progress(f"ScanCode reported {len(report.get('files', []))} files") return report From 7b4ac3775a9676c61d19ae6aaaad338b4c391f2b Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:03:16 +0000 Subject: [PATCH 23/24] Update pinned SBOM generator to latest image --- .github/workflows/bake_targets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index bf1ee1b4..59b2d61c 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -66,7 +66,7 @@ jobs: registry: ghcr.io/${{ github.repository_owner }} revision: ${{ github.sha }} # renovate: datasource=docker depName=ghcr.io/cnpg-extensions/cnpg-sbom-generator - sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:c23d06ef1d48074c56fa473a134aa887b06be7e5d3ab43c1b06cce2f899eb29e + sbom_generator: ghcr.io/cnpg-extensions/cnpg-sbom-generator:latest@sha256:38605482cdeb890e015a0d0e49edd473ecb483a2c0f075f3550e4470f247861e with: # Use the checkout so Bake sees the injected Dockerfile declaration. source: . From 07a6c3cc1bb22d7f211d452426e9e7a99c39de0a Mon Sep 17 00:00:00 2001 From: "ardentperf-agent[bot]" <265149240+ardentperf-agent[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:33:14 +0000 Subject: [PATCH 24/24] Restore upstream GitHub Action pins after rebase --- .github/workflows/bake_targets.yml | 4 ++-- .github/workflows/sbom-generator.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bake_targets.yml b/.github/workflows/bake_targets.yml index 59b2d61c..75f3b0dc 100644 --- a/.github/workflows/bake_targets.yml +++ b/.github/workflows/bake_targets.yml @@ -39,7 +39,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 with: platforms: 'linux/arm64' @@ -58,7 +58,7 @@ jobs: sed -i '2i ARG BUILDKIT_SBOM_SCAN_STAGE=builder' "$EXTENSION/Dockerfile" - name: Build and push - uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7 + uses: docker/bake-action@018cb6412ab401ebaa809aa5f85966b74628600f # v7 id: build env: BUILDX_METADATA_PROVENANCE: disabled diff --git a/.github/workflows/sbom-generator.yml b/.github/workflows/sbom-generator.yml index 66ba1fe1..7b42623f 100644 --- a/.github/workflows/sbom-generator.yml +++ b/.github/workflows/sbom-generator.yml @@ -40,12 +40,12 @@ jobs: run: python3 -m unittest discover -s sbom-generator/tests -p 'test_*.py' - name: Set up QEMU - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 with: platforms: linux/arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4 - name: Log in to GHCR if: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish == true) }}