From d1d914b4c36d0ad556f0ed4da7f22d802d66093c Mon Sep 17 00:00:00 2001 From: Vince Perri <5596945+vinceaperri@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:44:54 +0000 Subject: [PATCH 1/3] update --- .pipelines/github-pr-validation.yml | 1 + acl/tests/kola_enforcing.yaml | 2 + build_library/build_image_util.sh | 8 + build_library/prod_image_util.sh | 6 +- build_library/reports_util.sh | 53 ++ build_library/rpm/build_image_util.sh | 29 + .../rpm/generate_package_manifest.py | 556 ++++++++++++++++++ build_library/rpm/rpm_install.sh | 8 +- build_library/rpm/tests/requirements.txt | 8 + .../tests/test_generate_package_manifest.sh | 92 +++ .../testdata/expected-manifest.spdx.json | 519 ++++++++++++++++ .../rpm/tests/testdata/nevra-packages.txt | 21 + .../rpm/tests/testdata/tdnf-installed.txt | 21 + .../rpm/tests/validate_golden_manifest.sh | 59 ++ build_sysext | 55 +- 15 files changed, 1425 insertions(+), 13 deletions(-) create mode 100755 build_library/rpm/generate_package_manifest.py create mode 100644 build_library/rpm/tests/requirements.txt create mode 100755 build_library/rpm/tests/test_generate_package_manifest.sh create mode 100644 build_library/rpm/tests/testdata/expected-manifest.spdx.json create mode 100644 build_library/rpm/tests/testdata/nevra-packages.txt create mode 100644 build_library/rpm/tests/testdata/tdnf-installed.txt create mode 100755 build_library/rpm/tests/validate_golden_manifest.sh diff --git a/.pipelines/github-pr-validation.yml b/.pipelines/github-pr-validation.yml index 6c611ebaca3..c3cd003e98c 100644 --- a/.pipelines/github-pr-validation.yml +++ b/.pipelines/github-pr-validation.yml @@ -46,3 +46,4 @@ extends: parameters: aclRef: $(Build.SourceBranch) mantleRef: 'aclmain' + testGeneratePackageManifest: true diff --git a/acl/tests/kola_enforcing.yaml b/acl/tests/kola_enforcing.yaml index 28e1c8667bf..cc806cfd7e4 100644 --- a/acl/tests/kola_enforcing.yaml +++ b/acl/tests/kola_enforcing.yaml @@ -109,6 +109,8 @@ tests: - name: acl.ignition.v1.users - name: acl.ignition.v2.users + - name: acl.packages.package-manifest + - name: bpf.execsnoop exceptions: - imageVariants: [acl-t] diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index bf7f4dbba1b..b7c706a6eaa 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -830,6 +830,14 @@ EOF else # Skip this check in RPM mode - we intentionally populate ${DISTRO_SHARE_DIR}/etc earlier finish_image_backup_etc_rpm "${root_fs_dir}" + + if [[ "${IMAGE_BUILD_TYPE}" != "container" ]]; then + # Writes the image's package list and SPDX package manifest from the final rpmdb. + # + # Must be called after finish_image_backup_etc_rpm uninstalls the azurelinux-repos* packages above, + # and before the rootfs state loop deletes /var (and the rpmdb) below. + finish_image_package_manifest_rpm "${root_fs_dir}" "${image_name%.bin}" + fi fi # Remove the rootfs state as it should be recreated through the diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 1fa25905563..26753cdc926 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -130,7 +130,11 @@ create_prod_image() { sudo rm -rf "${root_sysext_mergedir}" fi - write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" + # In RPM mode finish_image_package_manifest_rpm writes this from the final + # rpmdb, after finish_image uninstalls the azurelinux-repos* packages. + if [[ "${PACKAGE_SOURCE_MODE}" == "PORTAGE" ]]; then + write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" + fi insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" insert_extra_slsa "${root_fs_dir}" diff --git a/build_library/reports_util.sh b/build_library/reports_util.sh index 3907c4f66cb..60f890b4631 100644 --- a/build_library/reports_util.sh +++ b/build_library/reports_util.sh @@ -120,6 +120,59 @@ write_disk_space_usage() { write_disk_space_usage_in_paths "${1}" "${2}" ./boot ./usr ./ } +# Where the SPDX package manifests are written to in a rootfs. +# systemd-sysext merges every sysext's copy of this directory over the image's, +# so a booted machine sees one directory holding the image's manifest and one +# per merged sysext. +OS_MANIFESTS_DIR="/usr/share/os-manifests" + +# Usage: +# +# write_package_manifest image "${root}" "${name}" "${version}" "${packages_file}" "${created_epoch}" +# write_package_manifest sysext "${root}" "${name}" "${version}" "${packages_file}" "${created_epoch}" +# +# The kind picks the filename. An image writes package-manifest.spdx.json and a +# sysext writes package-manifest..spdx.json, so the copies merged into one +# /usr safely. +# +# The document is not validated here. Its shape is a property of the generator, +# not of any one rootfs, so it is checked against a fixture and a golden SPDX +# 2.2 document by the Build RPMs job of the ACL GitHub PR pipeline, which runs +# /build_library/rpm/tests/test_generate_package_manifest.sh. +write_package_manifest() { + local kind="${1}" + local root="${2}" + local name="${3}" + local version="${4}" + local packages_file="${5}" + local created_epoch="${6}" + + local infix + case "${kind}" in + image) infix="" ;; + sysext) infix=".${name}" ;; + *) die "write_package_manifest: expected kind 'image' or 'sysext', got '${kind}'" ;; + esac + + local output="${root}${OS_MANIFESTS_DIR}/package-manifest${infix}.spdx.json" + + info "Writing ${output##*/}" + + # build_image runs as the sdk user, so writing into an image rootfs needs sudo. + sudo install -d -m 0755 "${output%/*}" + + # --force because BUILD_DIR is caller-supplied. + sudo "${BUILD_LIBRARY_DIR}/rpm/generate_package_manifest.py" \ + --packages-file="${packages_file}" \ + --manifest-file="${output}" \ + --manifest-name="${name}" \ + --manifest-version="${version}" \ + --created-epoch="${created_epoch}" \ + --force + + sudo chmod 0644 "${output}" +} + # Write an SPDX SBOM for a rootfs tree. write_sysext_sbom() { local rootfs="${1}"; shift diff --git a/build_library/rpm/build_image_util.sh b/build_library/rpm/build_image_util.sh index 4aadbcc7bbd..1b370f79279 100644 --- a/build_library/rpm/build_image_util.sh +++ b/build_library/rpm/build_image_util.sh @@ -1451,6 +1451,35 @@ finish_image_backup_etc_rpm() { sudo cp -a "${root_fs_dir}/etc" "${ETC_FULL_PATH}" } +# Write the image's package list and SPDX manifest from the final rpmdb. +finish_image_package_manifest_rpm() { + local root_fs_dir="$1" + local image_base_name="$2" + + if [[ -z "${BUILD_DIR:-}" ]]; then + die "RPM mode: BUILD_DIR is not set — cannot write the image package list" + fi + + local packages_file="${BUILD_DIR}/${image_base_name}_packages.txt" + + info "RPM mode: Writing ${packages_file##*/}" + rpm_query_packages "${root_fs_dir}" > "${packages_file}" + if [[ ! -s "${packages_file}" ]]; then + die "RPM mode: No packages in ${root_fs_dir}" + fi + + local created_epoch + created_epoch=$(stat -c '%Y' "${root_fs_dir}/usr/lib/os-release") + + write_package_manifest \ + image \ + "${root_fs_dir}" \ + "${image_base_name}" \ + "${IMAGE_VERSION_ID}${IMAGE_BUILD_ID:++${IMAGE_BUILD_ID}}" \ + "${packages_file}" \ + "${created_epoch}" +} + # Escape a string for JSON - handles quotes, backslashes, and control characters json_escape() { local str="$1" diff --git a/build_library/rpm/generate_package_manifest.py b/build_library/rpm/generate_package_manifest.py new file mode 100755 index 00000000000..3b2b9aa478f --- /dev/null +++ b/build_library/rpm/generate_package_manifest.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 + +import argparse +import dataclasses +import datetime +import hashlib +import json +import os +import re +import sys +import urllib.parse +import uuid +from collections.abc import Callable +from typing import Any + +# spdxVersion field value. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#61-spdx-version-field- +SPDX_VERSION = "SPDX-2.2" + +# dataLicense field value. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#62-data-license-field- +DATA_LICENSE = "CC0-1.0" + +# SPDXID field value for the document itself. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#63-spdx-identifier-field- +SPDXID = "SPDXRef-DOCUMENT" + +# SPDXID field value for the package standing in for the artifact as a whole. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/package-information.md#72-package-spdx-identifier-field- +ROOT_SPDXID = "SPDXRef-DocumentRoot" + +# SPDXID field value for each package. The "SPDXRef-" prefix is mandated and what +# follows it has to be unique across every element in the document, so index is +# included, since package names alone cannot guarantee this. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/package-information.md#72-package-spdx-identifier-field- +PACKAGE_SPDXID_FORMAT = "SPDXRef-Package-{index}-{name}" + +# SPDXID field value's validation mechanism for packages. This regex is used to +# replace invalid characters in a package name with a dash, so the resulting +# SPDXID is a valid "idstring" per the spec. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/package-information.md#72-package-spdx-identifier-field- +NON_SPDXID = re.compile(r"[^A-Za-z0-9.\-]") + +# documentNamespace field value base. It doesn't need to be a resolvable URL. +# It just needs to be a unique URI for the document. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#65-spdx-document-namespace-field- +DOCUMENT_NAMESPACE_BASE = "https://azurelinux.microsoft.com/spdxdocs" + +# This regex validates --manifest-name, which becomes a path segment of documentNamespace. +# The spec requires to be an absolute RFC 3986 URI, "#" excepted. +# This regex admits exactly what RFC 3986 allows in a path segment. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#65-spdx-document-namespace-field- +# -- https://www.rfc-editor.org/rfc/rfc3986#section-3.3 +MANIFEST_NAME_RE = re.compile(r"(?:[A-Za-z0-9\-._~!$&'()*+,;=:@]|%[0-9A-Fa-f]{2})+") + +# externalRefs[0].referenceCategory field value for each package. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/package-information.md#721-external-reference-field- +REFERENCE_CATEGORY = "PACKAGE-MANAGER" + +# externalRefs[0].referenceType field value. Describes the reference locator. +# Annex F registers purl under the PACKAGE-MANAGER category. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/external-repository-identifiers.md#f35-purl- +REFERENCE_TYPE = "purl" + +# externalRefs[0].referenceLocator field value's namespace for each package. +# Reference locators are RPM-type PURLs, and their namespace is the vendor. This +# value is taken from Azure Container Linux's os-release ID. +# -- https://github.com/package-url/purl-spec/blob/main/docs/types/definitions/rpm-definition.md#namespace-definition +PURL_NAMESPACE = "azurelinux" + +# relationships[].relationshipType field value tying the document to the root +# package. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/relationships-between-SPDX-elements.md#111-relationship-field- +DESCRIBES_RELATIONSHIP_TYPE = "DESCRIBES" + +# relationships[].relationshipType field value tying each package to the root. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/relationships-between-SPDX-elements.md#111-relationship-field- +CONTAINS_RELATIONSHIP_TYPE = "CONTAINS" + +# supplier field value for each package. The "Organization: " prefix is needed. +# The field names the distribution source, i.e. PMC. Optional in SPDX 2.2, but +# required for NTIA conformance. +# -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/package-information.md#75-package-supplier-field- +# -- https://www.ntia.gov/sites/default/files/publications/sbom_minimum_elements_report_0.pdf +SUPPLIER = "Organization: Microsoft Corporation" + +# Stands in for a field the document creator has not determined. +NOASSERTION = "NOASSERTION" + +# The two arches ACL builds plus noarch. +VALID_ARCHES = {"x86_64", "aarch64", "noarch"} + +# rpmdb pseudo-entries for imported signing keys, which are flagged as invalid +# packages. This cannot match real package NEVRAs, which always end in ".". +GPG_PUBKEY_RE = re.compile(r"^gpg-pubkey-[0-9a-f-]+\Z") + +# --packages-format values, each named for the command whose output it reads. +PACKAGES_FORMAT_NEVRA = "nevra" +PACKAGES_FORMAT_TDNF = "tdnf" +PACKAGES_FORMATS = [ + PACKAGES_FORMAT_NEVRA, + PACKAGES_FORMAT_TDNF, +] + +# Column count of a `tdnf list installed` line. +TDNF_COLUMNS = 3 + +# This regex validates --created-epoch. +CREATED_EPOCH_RE = re.compile(r"[0-9]+") + + +@dataclasses.dataclass(frozen=True) +class Package: + name: str + epoch: str | None + version: str + release: str + arch: str + + @property + def evr(self) -> str: + epoch = f"{self.epoch}:" if self.epoch else "" + return f"{epoch}{self.version}-{self.release}" + + @property + def evra(self) -> str: + return f"{self.evr}.{self.arch}" + + @property + def nevra(self) -> str: + return f"{self.name}-{self.evra}" + + +def create_package(name: str, epoch: str, version: str, release: str, arch: str) -> Package: + """Strip and validate raw NEVRA components, then build the Package.""" + name = name.strip() + epoch = epoch.strip() + version = version.strip() + release = release.strip() + arch = arch.strip() + + if not name: + raise ValueError("empty package name") + + if not version: + raise ValueError("empty package version") + + if not release: + raise ValueError("empty package release") + + if arch not in VALID_ARCHES: + raise ValueError( + f"unrecognized architecture " + f"[arch={arch!r}, expected one of {sorted(VALID_ARCHES)}]" + ) + + # Epoch tag 1003: "An absent epoch is equal to epoch value 0". + # https://rpm.org/docs/latest/manual/tags.html + return Package( + name=name, + epoch=epoch if epoch and epoch != "0" else None, + version=version, + release=release, + arch=arch, + ) + + +def validate_nevra_entry(entry: str) -> None: + """Reject anything in `rpm -qa` output that is not a package.""" + if GPG_PUBKEY_RE.fullmatch(entry): + raise ValueError("imported signing key") + + +def parse_nevra_package(nevra: str) -> Package: + """`rpm -qa --qf '%{NEVRA}\\n'` line -> Package. + + The lines are -[:]-., which is the + NEVRA format. + """ + nevr, _, arch = nevra.rpartition(".") + nev, _, release = nevr.rpartition("-") + name, _, ev = nev.rpartition("-") + epoch, _, version = ev.rpartition(":") + + return create_package( + name=name, + epoch=epoch, + version=version, + release=release, + arch=arch, + ) + + +def parse_tdnf_package(line: str) -> Package: + """`tdnf list installed` line -> Package. + + The columns are ., <[epoch:]version-release> and @, + space-padded to align. An image carrying tdnf but no rpm binary can still + produce this. + """ + columns = line.split() + if len(columns) != TDNF_COLUMNS: + raise ValueError( + f"expected '. <[epoch:]version-release> @' " + f"[columns={len(columns)}, expected {TDNF_COLUMNS}]" + ) + + name_arch, evr, _repo = columns + + name, _, arch = name_arch.rpartition(".") + ev, _, release = evr.rpartition("-") + epoch, _, version = ev.rpartition(":") + + return create_package( + name=name, + epoch=epoch, + version=version, + release=release, + arch=arch, + ) + + +def read_entries(packages_file: str) -> list[tuple[int, str]]: + """Line number and stripped content of every line in a package list.""" + entries: list[tuple[int, str]] = [] + + with open(packages_file, "r", encoding="utf-8") as f: + for number, line in enumerate(f, 1): + entry = line.strip() + if entry: + entries.append((number, entry)) + + return entries + + +def read_packages( + packages_file: str, + validate: Callable[[str], None] | None, + parse: Callable[[str], Package], +) -> list[Package]: + seen: set[Package] = set() + packages: list[Package] = [] + + for number, entry in read_entries(packages_file): + try: + if validate is not None: + validate(entry) + + package = parse(entry) + except ValueError as error: + raise ValueError(f"{packages_file}:{number}: {error} [line={entry!r}]") from error + + if package in seen: + raise ValueError(f"{packages_file}:{number}: duplicate package [line={entry!r}]") + + seen.add(package) + packages.append(package) + + return packages + + +def purl_encode(component_data: str) -> str: + """Percent-encode purl component data. + + purl's "allowed set" is the alphanumerics plus ".-_~", which is exactly what + quote() leaves alone. Everything else has to be a triplet, so the "+" in a + name like libstdc++ becomes "%2B". + -- https://github.com/package-url/purl-spec/blob/main/docs/specification/standard/specification.md#character-encoding + """ + return urllib.parse.quote(component_data, safe="") + + +def package_url(package: Package) -> str: + """Package URL for an RPM, per the purl rpm type.""" + qualifiers = [f"arch={purl_encode(package.arch)}"] + if package.epoch: + qualifiers.append(f"epoch={purl_encode(package.epoch)}") + + name = purl_encode(package.name) + version = purl_encode(f"{package.version}-{package.release}") + + return f"pkg:rpm/{PURL_NAMESPACE}/{name}@{version}?{'&'.join(qualifiers)}" + + +def created(created_epoch: int) -> str: + """Build creationInfo.created, which the spec fixes to UTC to the second. + + -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#69-created-field- + """ + return datetime.datetime.fromtimestamp(created_epoch, tz=datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def creator() -> str: + """Build creationInfo.creators, which is spec'd as "toolidentifier-version". + + This script ships with the repo and has no release of its own, so the version + is a digest of its own source. A hand-maintained constant would drift. + -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#68-creator-field- + """ + name = os.path.splitext(os.path.basename(__file__))[0] + + with open(__file__, "rb") as f: + version = hashlib.sha256(f.read()).hexdigest() + + return f"Tool: {name}-{version}" + + +def document_namespace(manifest_name: str, manifest_version: str, packages: list[Package]) -> str: + """Build the documentNamespace URI. + + Follows the recommended [CreatorWebsite]/[pathToSpdx]/[DocumentName]-[UUID] + shape, building a version 5 UUID over the name, the version and the package + list so that the same list always yields the same URI. + -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#65-spdx-document-namespace-field- + """ + nevras = "/".join(package.nevra for package in packages) + namespace = f"{DOCUMENT_NAMESPACE_BASE}/{manifest_name}/{manifest_version}/{nevras}" + unique = uuid.uuid5(uuid.NAMESPACE_URL, namespace) + + return f"{DOCUMENT_NAMESPACE_BASE}/{manifest_name}-{unique}" + + +def create_package_manifest( + manifest_name: str, + manifest_version: str, + packages: list[Package], + created_epoch: int, +) -> dict[str, Any]: + spdx_packages: list[dict[str, Any]] = [ + { + "SPDXID": ROOT_SPDXID, + "name": manifest_name, + "versionInfo": manifest_version, + "supplier": SUPPLIER, + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": NOASSERTION, + "copyrightText": NOASSERTION, + } + ] + spdx_package_ids: list[str] = [] + + for index, package in enumerate(packages): + spdx_package_id = PACKAGE_SPDXID_FORMAT.format( + index=index, + name=NON_SPDXID.sub("-", package.name) + ) + spdx_package_ids.append(spdx_package_id) + spdx_packages.append( + { + "SPDXID": spdx_package_id, + "name": package.name, + "versionInfo": package.evr, + "supplier": SUPPLIER, + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": NOASSERTION, + "copyrightText": NOASSERTION, + "externalRefs": [ + { + "referenceCategory": REFERENCE_CATEGORY, + "referenceType": REFERENCE_TYPE, + "referenceLocator": package_url(package), + } + ], + } + ) + + relationships: list[dict[str, str]] = [ + { + "spdxElementId": SPDXID, + "relatedSpdxElement": ROOT_SPDXID, + "relationshipType": DESCRIBES_RELATIONSHIP_TYPE, + } + ] + relationships.extend( + { + "spdxElementId": ROOT_SPDXID, + "relatedSpdxElement": spdx_id, + "relationshipType": CONTAINS_RELATIONSHIP_TYPE, + } + for spdx_id in spdx_package_ids + ) + + return { + "spdxVersion": SPDX_VERSION, + "dataLicense": DATA_LICENSE, + "SPDXID": SPDXID, + "name": manifest_name, + "documentNamespace": document_namespace(manifest_name, manifest_version, packages), + "creationInfo": { + "created": created(created_epoch), + "creators": [creator()], + }, + "packages": spdx_packages, + "documentDescribes": [ROOT_SPDXID], + "relationships": relationships, + } + + +def write_json(path: str, document: dict[str, Any]) -> None: + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(document, indent=2, sort_keys=True) + "\n") + + +def validate_input_file(value: str) -> str: + if not os.path.isfile(value): + raise argparse.ArgumentTypeError( + f"expected an existing file [value={value!r}]" + ) + + return value + + +def validate_manifest_name(value: str) -> str: + if not MANIFEST_NAME_RE.fullmatch(value): + raise argparse.ArgumentTypeError( + f"expected a non-empty URI path segment [value={value!r}]" + ) + + return value + + +def validate_manifest_version(value: str) -> str: + if not value.strip(): + raise argparse.ArgumentTypeError( + f"expected a non-empty version [value={value!r}]" + ) + + return value + + +def validate_created_epoch(value: str) -> int: + if not CREATED_EPOCH_RE.fullmatch(value): + raise argparse.ArgumentTypeError( + f"expected a non-negative Unix timestamp [value={value!r}]" + ) + + return int(value) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate an SPDX 2.2 package manifest from an installed package list." + ) + + parser.add_argument( + "--packages-file", + required=True, + type=validate_input_file, + help="The path to the package list to convert, one package per line. (required)", + ) + + parser.add_argument( + "--packages-format", + default=PACKAGES_FORMAT_NEVRA, + choices=PACKAGES_FORMATS, + help=( + f"The format of --packages-file. " + f"'{PACKAGES_FORMAT_NEVRA}' is one NEVRA per line. " + f"'{PACKAGES_FORMAT_TDNF}' is `tdnf list installed` output, for an image " + f"carrying no rpm binary. (optional, default: {PACKAGES_FORMAT_NEVRA})" + ), + ) + + parser.add_argument( + "--manifest-file", + required=True, + help=( + "The path to the SPDX document to generate. Its parent directory must already exist. (required)" + ), + ) + + parser.add_argument( + "--manifest-name", + required=True, + type=validate_manifest_name, + help="The SPDX document name, e.g. the image or sysext name. (required)", + ) + + parser.add_argument( + "--manifest-version", + required=True, + type=validate_manifest_version, + help=( + "The version of the image or sysext being described. Becomes the " + "root package's versionInfo. NTIA's minimum elements require a " + "version for every component, and the root is one. (required)" + ), + ) + + parser.add_argument( + "--created-epoch", + required=True, + type=validate_created_epoch, + help="The Unix timestamp for the document's created field. (required)", + ) + + parser.add_argument( + "--force", + action="store_true", + help="Overwrite the SPDX document if it already exists. (optional)", + ) + + args = parser.parse_args() + + if not args.force and os.path.lexists(args.manifest_file): + parser.error( + f"manifest file already exists, pass --force to overwrite it " + f"[value={args.manifest_file!r}]" + ) + + return args + + +def main() -> int: + args = parse_args() + packages_file: str = args.packages_file + packages_format: str = args.packages_format + manifest_file: str = args.manifest_file + manifest_name: str = args.manifest_name + manifest_version: str = args.manifest_version + created_epoch: int = args.created_epoch + + if packages_format == PACKAGES_FORMAT_TDNF: + validate, parse = None, parse_tdnf_package + else: + validate, parse = validate_nevra_entry, parse_nevra_package + + try: + packages = read_packages(packages_file, validate, parse) + except ValueError as error: + print( + f"{os.path.basename(__file__)}: failed to read packages: {error}", + file=sys.stderr, + ) + return 1 + + # Guarantee a stable order for deterministic manifest generation. + packages = sorted(packages, key=lambda package: package.nevra) + + manifest = create_package_manifest( + manifest_name, + manifest_version, + packages, + created_epoch, + ) + write_json(manifest_file, manifest) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_library/rpm/rpm_install.sh b/build_library/rpm/rpm_install.sh index b442905a2b4..5f5d626f8c3 100644 --- a/build_library/rpm/rpm_install.sh +++ b/build_library/rpm/rpm_install.sh @@ -534,7 +534,13 @@ rpm_query_packages() { # 2. to include the epoch: "rpm -qa" returns NVRAs, not NEVRAs, for packages, and the epoch is sometimes needed # to completely identify a package, e.g. during security scanning, where the epoch is used to map packages to # vulnerabilities. - sudo rpm --dbpath="${dbpath}" -qa --qf '%|ARCH?{%{NEVRA}\n}:{}|' 2>/dev/null | sort + # + # Captured rather than piped straight into sort so rpm failures aren't masked. + local nevras + nevras=$(sudo rpm --dbpath="${dbpath}" -qa --qf '%|ARCH?{%{NEVRA}\n}:{}|') || return 1 + + [[ -n "${nevras}" ]] || return 0 + sort <<< "${nevras}" } # Get RPM package metadata diff --git a/build_library/rpm/tests/requirements.txt b/build_library/rpm/tests/requirements.txt new file mode 100644 index 00000000000..f1f06a7ed25 --- /dev/null +++ b/build_library/rpm/tests/requirements.txt @@ -0,0 +1,8 @@ +# Dependencies of ./validate_golden_manifest.sh +# +# To bump, install the two direct packages at the versions you want and freeze: +# python3 -m venv .venv +# .venv/bin/pip install spdx-tools== ntia-conformance-checker== +# .venv/bin/pip freeze > requirements.txt +ntia_conformance_checker==5.0.3 +spdx-tools==0.8.5 diff --git a/build_library/rpm/tests/test_generate_package_manifest.sh b/build_library/rpm/tests/test_generate_package_manifest.sh new file mode 100755 index 00000000000..4781d57ebe0 --- /dev/null +++ b/build_library/rpm/tests/test_generate_package_manifest.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Check that generate_package_manifest.py emits the expected SPDX 2.2 document. +# +# Usage: test_generate_package_manifest.sh [workdir] + +set -euo pipefail + +TESTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GENERATOR="${TESTS_DIR}/../generate_package_manifest.py" + +# Conformance is a property of the generator, not of any one rootfs, so this is +# run against a fixture instead of inside the image build. This file is a real +# capture from an ACL production image, trimmed, plus one epoch-bearing entry +# and one non-native arch. +NEVRA_PACKAGES_FILE="${TESTS_DIR}/testdata/nevra-packages.txt" + +# The same package set as nevra-packages.txt, in `tdnf list installed` layout. +# ACL-T has no rpm binary and feeds the generator this instead, so the two files +# describing one package set is what makes the cross-check below meaningful. +TDNF_PACKAGES_FILE="${TESTS_DIR}/testdata/tdnf-installed.txt" + +# When a change to the emitted document is intentional, pass to the +# script, copy the normalized output to the golden file, then re-validate it: +# cp "${WORK_DIR}/package-manifest.spdx.json.normalized" testdata/expected-manifest.spdx.json +# ./validate_golden_manifest.sh +GOLDEN_FILE="${TESTS_DIR}/testdata/expected-manifest.spdx.json" + +# Keep a caller-supplied so its output can be inspected, or clean up +# one we made ourselves so we don't leave scratch files behind on a local run. +if [[ $# -ge 1 ]]; then + WORK_DIR="${1}" +else + WORK_DIR="$(mktemp -d)" + trap 'rm -rf "${WORK_DIR}"' EXIT +fi +mkdir -p "${WORK_DIR}" + +MANIFEST="${WORK_DIR}/package-manifest.spdx.json" +MANIFEST_AGAIN="${MANIFEST}.again" +MANIFEST_NORMALIZED="${MANIFEST}.normalized" +MANIFEST_TDNF="${MANIFEST}.tdnf" + +MANIFEST_NAME="acl_production_image" +MANIFEST_VERSION="0.0.0-spec-conformance" +CREATED_EPOCH=1735689600 + +echo "=== Generating manifest from ${NEVRA_PACKAGES_FILE##*/} ===" +"${GENERATOR}" \ + --packages-file="${NEVRA_PACKAGES_FILE}" \ + --manifest-file="${MANIFEST}" \ + --manifest-name="${MANIFEST_NAME}" \ + --manifest-version="${MANIFEST_VERSION}" \ + --created-epoch="${CREATED_EPOCH}" \ + --force + +echo "=== Checking the generator is byte-identical on a second run ===" +"${GENERATOR}" \ + --packages-file="${NEVRA_PACKAGES_FILE}" \ + --manifest-file="${MANIFEST_AGAIN}" \ + --manifest-name="${MANIFEST_NAME}" \ + --manifest-version="${MANIFEST_VERSION}" \ + --created-epoch="${CREATED_EPOCH}" \ + --force +cmp "${MANIFEST}" "${MANIFEST_AGAIN}" + +echo "=== Comparing against ${GOLDEN_FILE##*/} ===" +sed -E 's|("Tool: generate_package_manifest)-[0-9a-f]{64}"|\1"|' \ + "${MANIFEST}" > "${MANIFEST_NORMALIZED}" +diff -u "${GOLDEN_FILE}" "${MANIFEST_NORMALIZED}" + +# libstdc++ is the fixture's canary for purl encoding: '+' is outside purl's +# allowed set, so a canonical locator has to carry %2B instead. +EXPECTED_PURL='pkg:rpm/azurelinux/libstdc%2B%2B@13.2.0-7.azl3?arch=x86_64' +echo "=== Checking ${EXPECTED_PURL} is emitted canonically ===" +if ! grep -qF "\"referenceLocator\": \"${EXPECTED_PURL}\"" "${MANIFEST}"; then + echo "referenceLocator is not the canonical purl: ${EXPECTED_PURL}" >&2 + exit 1 +fi + +echo "=== Checking --packages-format=tdnf describes the same package set ===" +"${GENERATOR}" \ + --packages-file="${TDNF_PACKAGES_FILE}" \ + --packages-format=tdnf \ + --manifest-file="${MANIFEST_TDNF}" \ + --manifest-name="${MANIFEST_NAME}" \ + --manifest-version="${MANIFEST_VERSION}" \ + --created-epoch="${CREATED_EPOCH}" \ + --force +diff -u "${MANIFEST}" "${MANIFEST_TDNF}" + +echo "=== PASS ===" diff --git a/build_library/rpm/tests/testdata/expected-manifest.spdx.json b/build_library/rpm/tests/testdata/expected-manifest.spdx.json new file mode 100644 index 00000000000..1b1e6d5f754 --- /dev/null +++ b/build_library/rpm/tests/testdata/expected-manifest.spdx.json @@ -0,0 +1,519 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "creationInfo": { + "created": "2025-01-01T00:00:00Z", + "creators": [ + "Tool: generate_package_manifest" + ] + }, + "dataLicense": "CC0-1.0", + "documentDescribes": [ + "SPDXRef-DocumentRoot" + ], + "documentNamespace": "https://azurelinux.microsoft.com/spdxdocs/acl_production_image-03b29ff5-169e-5841-a76e-819f6cad8af7", + "name": "acl_production_image", + "packages": [ + { + "SPDXID": "SPDXRef-DocumentRoot", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "acl_production_image", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "0.0.0-spec-conformance" + }, + { + "SPDXID": "SPDXRef-Package-0-azurelinux-release", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/azurelinux-release@3.0-49.azl3?arch=noarch", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "azurelinux-release", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "3.0-49.azl3" + }, + { + "SPDXID": "SPDXRef-Package-1-bash", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/bash@5.2.15-3.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "bash", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "5.2.15-3.azl3" + }, + { + "SPDXID": "SPDXRef-Package-2-ca-certificates-shared", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/ca-certificates-shared@3.0.0-15.azl3?arch=noarch", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "ca-certificates-shared", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "3.0.0-15.azl3" + }, + { + "SPDXID": "SPDXRef-Package-3-coreutils", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/coreutils@9.4-7.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "coreutils", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "9.4-7.azl3" + }, + { + "SPDXID": "SPDXRef-Package-4-device-mapper-event-libs", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/device-mapper-event-libs@2.03.23-1.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "device-mapper-event-libs", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "2.03.23-1.azl3" + }, + { + "SPDXID": "SPDXRef-Package-5-e2fsprogs-libs", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/e2fsprogs-libs@1.47.0-2.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "e2fsprogs-libs", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "1.47.0-2.azl3" + }, + { + "SPDXID": "SPDXRef-Package-6-elfutils-default-yama-scope", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/elfutils-default-yama-scope@0.189-6.azl3?arch=noarch", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "elfutils-default-yama-scope", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "0.189-6.azl3" + }, + { + "SPDXID": "SPDXRef-Package-7-filesystem", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/filesystem@1.1-21.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "filesystem", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "1.1-21.azl3" + }, + { + "SPDXID": "SPDXRef-Package-8-glibc", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/glibc@2.38-20.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "glibc", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "2.38-20.azl3" + }, + { + "SPDXID": "SPDXRef-Package-9-iscsi-initiator-utils-iscsiuio", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/iscsi-initiator-utils-iscsiuio@2.1.9-1.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "iscsi-initiator-utils-iscsiuio", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "2.1.9-1.azl3" + }, + { + "SPDXID": "SPDXRef-Package-10-kernel", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/kernel@6.6.145.2-1.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "kernel", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "6.6.145.2-1.azl3" + }, + { + "SPDXID": "SPDXRef-Package-11-libstdc--", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/libstdc%2B%2B@13.2.0-7.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "libstdc++", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "13.2.0-7.azl3" + }, + { + "SPDXID": "SPDXRef-Package-12-lshw", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/lshw@B.02.20-1.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "lshw", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "B.02.20-1.azl3" + }, + { + "SPDXID": "SPDXRef-Package-13-openssl-libs", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/openssl-libs@3.3.7-4.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "openssl-libs", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "3.3.7-4.azl3" + }, + { + "SPDXID": "SPDXRef-Package-14-python3-libs", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/python3-libs@3.12.9-6.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "python3-libs", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "3.12.9-6.azl3" + }, + { + "SPDXID": "SPDXRef-Package-15-shadow-utils-subid", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/shadow-utils-subid@4.14.3-2.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "shadow-utils-subid", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "4.14.3-2.azl3" + }, + { + "SPDXID": "SPDXRef-Package-16-systemd", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/systemd@255-33.azl3?arch=x86_64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "systemd", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "255-33.azl3" + }, + { + "SPDXID": "SPDXRef-Package-17-systemd-rpm-macros", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/systemd-rpm-macros@255-33.azl3?arch=noarch", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "systemd-rpm-macros", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "255-33.azl3" + }, + { + "SPDXID": "SPDXRef-Package-18-tzdata", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/tzdata@2025c-2.azl3?arch=noarch", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "tzdata", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "2025c-2.azl3" + }, + { + "SPDXID": "SPDXRef-Package-19-zlib", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/zlib@1.3.1-1.azl3?arch=aarch64", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "zlib", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "1.3.1-1.azl3" + }, + { + "SPDXID": "SPDXRef-Package-20-zlib", + "copyrightText": "NOASSERTION", + "downloadLocation": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceLocator": "pkg:rpm/azurelinux/zlib@1.3.1-1.azl3?arch=x86_64&epoch=1", + "referenceType": "purl" + } + ], + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "name": "zlib", + "supplier": "Organization: Microsoft Corporation", + "versionInfo": "1:1.3.1-1.azl3" + } + ], + "relationships": [ + { + "relatedSpdxElement": "SPDXRef-DocumentRoot", + "relationshipType": "DESCRIBES", + "spdxElementId": "SPDXRef-DOCUMENT" + }, + { + "relatedSpdxElement": "SPDXRef-Package-0-azurelinux-release", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-1-bash", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-2-ca-certificates-shared", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-3-coreutils", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-4-device-mapper-event-libs", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-5-e2fsprogs-libs", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-6-elfutils-default-yama-scope", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-7-filesystem", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-8-glibc", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-9-iscsi-initiator-utils-iscsiuio", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-10-kernel", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-11-libstdc--", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-12-lshw", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-13-openssl-libs", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-14-python3-libs", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-15-shadow-utils-subid", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-16-systemd", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-17-systemd-rpm-macros", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-18-tzdata", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-19-zlib", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + }, + { + "relatedSpdxElement": "SPDXRef-Package-20-zlib", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-DocumentRoot" + } + ], + "spdxVersion": "SPDX-2.2" +} diff --git a/build_library/rpm/tests/testdata/nevra-packages.txt b/build_library/rpm/tests/testdata/nevra-packages.txt new file mode 100644 index 00000000000..4b42f70d97c --- /dev/null +++ b/build_library/rpm/tests/testdata/nevra-packages.txt @@ -0,0 +1,21 @@ +azurelinux-release-3.0-49.azl3.noarch +bash-5.2.15-3.azl3.x86_64 +ca-certificates-shared-3.0.0-15.azl3.noarch +coreutils-9.4-7.azl3.x86_64 +device-mapper-event-libs-2.03.23-1.azl3.x86_64 +e2fsprogs-libs-1.47.0-2.azl3.x86_64 +elfutils-default-yama-scope-0.189-6.azl3.noarch +filesystem-1.1-21.azl3.x86_64 +glibc-2.38-20.azl3.x86_64 +iscsi-initiator-utils-iscsiuio-2.1.9-1.azl3.x86_64 +kernel-6.6.145.2-1.azl3.x86_64 +libstdc++-13.2.0-7.azl3.x86_64 +lshw-B.02.20-1.azl3.x86_64 +openssl-libs-3.3.7-4.azl3.x86_64 +python3-libs-3.12.9-6.azl3.x86_64 +shadow-utils-subid-4.14.3-2.azl3.x86_64 +systemd-255-33.azl3.x86_64 +systemd-rpm-macros-255-33.azl3.noarch +tzdata-2025c-2.azl3.noarch +zlib-1.3.1-1.azl3.aarch64 +zlib-1:1.3.1-1.azl3.x86_64 diff --git a/build_library/rpm/tests/testdata/tdnf-installed.txt b/build_library/rpm/tests/testdata/tdnf-installed.txt new file mode 100644 index 00000000000..cae2f5a2ea3 --- /dev/null +++ b/build_library/rpm/tests/testdata/tdnf-installed.txt @@ -0,0 +1,21 @@ +azurelinux-release.noarch 3.0-49.azl3 @System +bash.x86_64 5.2.15-3.azl3 @System +ca-certificates-shared.noarch 3.0.0-15.azl3 @System +coreutils.x86_64 9.4-7.azl3 @System +device-mapper-event-libs.x86_64 2.03.23-1.azl3 @System +e2fsprogs-libs.x86_64 1.47.0-2.azl3 @System +elfutils-default-yama-scope.noarch 0.189-6.azl3 @System +filesystem.x86_64 1.1-21.azl3 @System +glibc.x86_64 2.38-20.azl3 @System +iscsi-initiator-utils-iscsiuio.x86_64 2.1.9-1.azl3 @System +kernel.x86_64 6.6.145.2-1.azl3 @System +libstdc++.x86_64 13.2.0-7.azl3 @System +lshw.x86_64 B.02.20-1.azl3 @System +openssl-libs.x86_64 3.3.7-4.azl3 @System +python3-libs.x86_64 3.12.9-6.azl3 @System +shadow-utils-subid.x86_64 4.14.3-2.azl3 @System +systemd.x86_64 255-33.azl3 @System +systemd-rpm-macros.noarch 255-33.azl3 @System +tzdata.noarch 2025c-2.azl3 @System +zlib.aarch64 1.3.1-1.azl3 @System +zlib.x86_64 1:1.3.1-1.azl3 @System diff --git a/build_library/rpm/tests/validate_golden_manifest.sh b/build_library/rpm/tests/validate_golden_manifest.sh new file mode 100755 index 00000000000..f251ba55bc6 --- /dev/null +++ b/build_library/rpm/tests/validate_golden_manifest.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Validate the golden document against SPDX 2.2 and the NTIA minimum elements. +# +# Deliberately not in CI because it requires third-party Python packages, and +# test_generate_package_manifest.sh, which is in CI, already validates that the +# generator reproduces the golden document byte for byte. This, however, says +# nothing about whether that document is a valid one, so this script is run +# whenever it is regenerated. +# +# The NTIA minimum elements are the published floor for identifying a component +# well enough to map it to a vulnerability database. +# -- https://www.ntia.gov/sites/default/files/publications/sbom_minimum_elements_report_0.pdf +# +# Usage: validate_golden_manifest.sh [workdir] + +set -euo pipefail + +TESTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFORMANCE="${TESTS_DIR}/test_generate_package_manifest.sh" + +REQUIREMENTS="${TESTS_DIR}/requirements.txt" + +if [[ $# -ge 1 ]]; then + WORK_DIR="${1}" +else + WORK_DIR="$(mktemp -d)" + trap 'rm -rf "${WORK_DIR}"' EXIT +fi +mkdir -p "${WORK_DIR}" + +# Validating the generated document rather than the golden file itself, because +# the golden is normalized to drop the generator's source digest and so is not +# the shape that ships. This also fails first if the two have diverged. +"${CONFORMANCE}" "${WORK_DIR}" + +MANIFEST="${WORK_DIR}/package-manifest.spdx.json" +VENV="${WORK_DIR}/.venv" + +echo "=== Provisioning validators ===" +python3 -m venv "${VENV}" +"${VENV}/bin/pip" install \ + --quiet \ + --disable-pip-version-check \ + --requirement "${REQUIREMENTS}" + +# pyspdxtools prints the individual validation messages. ntia-checker re-runs +# the same spdx-tools validation internally but reports only a pass/fail +# verdict, so running pyspdxtools first is what makes a failure diagnosable. +echo "=== SPDX 2.2 validation ===" +"${VENV}/bin/pyspdxtools" --infile="${MANIFEST}" + +# Both flags are already the defaults, pinned so a new release cannot move +# them: --comply cannot be fsct3-min since that needs per-package licences we +# do not have, and --sbom-spec must be spdx2 since that is what was built. +echo "=== NTIA minimum elements ===" +"${VENV}/bin/ntia-checker" --comply=ntia --sbom-spec=spdx2 "${MANIFEST}" + +echo "=== PASS ===" diff --git a/build_sysext b/build_sysext index de483061ff1..4ae1ebfc445 100755 --- a/build_sysext +++ b/build_sysext @@ -269,8 +269,12 @@ for package; do fi done -# Make squashfs generation more reproducible. -export SOURCE_DATE_EPOCH=$(stat -c '%Y' "${BUILD_DIR}/fs-root/usr/lib/os-release") +# Pins mksquashfs timestamps and the package manifest's created field to one +# value per image build. os-release is rewritten on every build, so this is the +# build's stamp time rather than a fixed epoch. Split from the export so set -e +# still catches a failing stat. +SOURCE_DATE_EPOCH=$(stat -c '%Y' "${BUILD_DIR}/fs-root/usr/lib/os-release") +export SOURCE_DATE_EPOCH # Unmount in order to get rid of the overlay umount "${BUILD_DIR}/${FLAGS_install_root_basename}" @@ -321,13 +325,30 @@ if [[ -n "${FLAGS_manglefs_script}" ]]; then "${FLAGS_manglefs_script}" "${BUILD_DIR}/${FLAGS_install_root_basename}" fi +# extension-release VERSION_ID must match the host os-release VERSION_ID. +# In RPM mode that's IMAGE_VERSION_ID, not FLATCAR_VERSION_ID. +_sysext_version_id="${FLATCAR_VERSION_ID}" +if [[ "${PACKAGE_SOURCE_MODE:-PORTAGE}" == "RPM" && -n "${IMAGE_VERSION_ID:-}" ]]; then + _sysext_version_id="${IMAGE_VERSION_ID}" +fi + # In RPM mode, write packages.txt after manglefs so that packages removed # by mangle scripts (e.g. containerd2/runc from docker) are excluded. if [[ "${PACKAGE_SOURCE_MODE}" == "RPM" ]]; then info "Writing ${SYSEXTNAME}_packages.txt" - # Create the file unconditionally; the diff below will overwrite it with - # actual contents when an RPM database is present. - touch "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" + # Create the files unconditionally; they'll be overwritten with actual contents + # when an RPM database is present. Truncate rather than touch since BUILD_DIR + # is caller-supplied and shared across sysexts, so may not be empty. + : > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" + : > "${BUILD_DIR}/.rpm-sysext-manifest.tmp" + + # Taking a snapshot is conditional on the base image having an rpmdb. Absent + # one, every sysext will diff to nothing, and without this check, would skip + # its manifest with no error. + if [[ ! -s "${BUILD_DIR}/.rpm-base.tmp" ]]; then + die "No pre-install package snapshot for ${SYSEXTNAME}; ${FLAGS_squashfs_base##*/} has no RPM database" + fi + # Diff against pre-install snapshot to list only sysext-added packages. if [[ -f "${BUILD_DIR}/${FLAGS_install_root_basename}/var/lib/rpm/rpmdb.sqlite" ]]; then rpm_query_packages "${BUILD_DIR}/${FLAGS_install_root_basename}" > "${BUILD_DIR}/.rpm-all.tmp" @@ -340,6 +361,24 @@ if [[ "${PACKAGE_SOURCE_MODE}" == "RPM" ]]; then else warn "No RPM database found for ${SYSEXTNAME}; ${SYSEXTNAME}_packages.txt will be empty" fi + + # Empty means either nothing was installed or the base image already had every + # package this sysext asks for. Both are valid sysexts with nothing of their + # own to describe. + if [[ -s "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" ]]; then + # _sysext_version_id is pinned to the host os-release VERSION_ID and so is + # only date-granular in the pipeline. The manifest is not bound by that + # contract and needs the build id to keep its documentNamespace unique. + write_package_manifest \ + sysext \ + "${BUILD_DIR}/${FLAGS_install_root_basename}" \ + "${SYSEXTNAME}" \ + "${_sysext_version_id}${IMAGE_BUILD_ID:++${IMAGE_BUILD_ID}}" \ + "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" \ + "${SOURCE_DATE_EPOCH}" + else + info "No packages unique to ${SYSEXTNAME}: skipping package manifest generation" + fi fi info "Removing non-/usr directories from sysext image" @@ -359,12 +398,6 @@ elif [[ "${PACKAGE_SOURCE_MODE}" == "RPM" ]]; then OS_ID="azurelinux" fi -# extension-release VERSION_ID must match the host os-release VERSION_ID. -# In RPM mode that's IMAGE_VERSION_ID, not FLATCAR_VERSION_ID. -_sysext_version_id="${FLATCAR_VERSION_ID}" -if [[ "${PACKAGE_SOURCE_MODE:-PORTAGE}" == "RPM" && -n "${IMAGE_VERSION_ID:-}" ]]; then - _sysext_version_id="${IMAGE_VERSION_ID}" -fi version_field="${VERSION_FIELD_OVERRIDE:-VERSION_ID=${_sysext_version_id}}" all_fields=( "ID=${OS_ID}" From cf7689d9732d3d74587f0cbf13e08ef6014a5b0e Mon Sep 17 00:00:00 2001 From: Vince Perri <5596945+vinceaperri@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:33:18 +0000 Subject: [PATCH 2/3] fix comment --- build_library/rpm/generate_package_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/rpm/generate_package_manifest.py b/build_library/rpm/generate_package_manifest.py index 3b2b9aa478f..83786b0f32f 100755 --- a/build_library/rpm/generate_package_manifest.py +++ b/build_library/rpm/generate_package_manifest.py @@ -310,7 +310,7 @@ def document_namespace(manifest_name: str, manifest_version: str, packages: list Follows the recommended [CreatorWebsite]/[pathToSpdx]/[DocumentName]-[UUID] shape, building a version 5 UUID over the name, the version and the package - list so that the same list always yields the same URI. + list, so the same three inputs always yield the same URI. -- https://github.com/spdx/spdx-spec/blob/development/v2.2.2/chapters/document-creation-information.md#65-spdx-document-namespace-field- """ nevras = "/".join(package.nevra for package in packages) From ddb1845798fb41cff99bbed1722729082e4dadcf Mon Sep 17 00:00:00 2001 From: Vince Perri <5596945+vinceaperri@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:46:57 +0000 Subject: [PATCH 3/3] do not enforce new test --- acl/tests/kola_enforcing.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acl/tests/kola_enforcing.yaml b/acl/tests/kola_enforcing.yaml index cc806cfd7e4..d4216c4ea7a 100644 --- a/acl/tests/kola_enforcing.yaml +++ b/acl/tests/kola_enforcing.yaml @@ -109,7 +109,8 @@ tests: - name: acl.ignition.v1.users - name: acl.ignition.v2.users - - name: acl.packages.package-manifest + # TEMPORARY (AB#23640): Enable acl.packages.package-manifest in kola_enforcing.yaml once the mantle LKG carries it + # - name: acl.packages.package-manifest - name: bpf.execsnoop exceptions: