From 49acd63e0dcde1824fdab3a81f7491409d82aa4e Mon Sep 17 00:00:00 2001 From: arpan Date: Mon, 14 Sep 2026 01:21:19 +0530 Subject: [PATCH] The build is reproducible, and CI checks that it is Every python -m build in a workflow now runs with SOURCE_DATE_EPOCH set to the commit's own timestamp, so the wheel is the same bytes wherever it is built; scripts/normalize_sdist.py does for the sdist what setuptools does not (copy times, uid, filesystem order, the gzip header), keeping every member and every byte of content. CI's package job builds a second time and diffs the hashes, so the property is checked on every pull request rather than claimed after a tag. CONTRIBUTING.md's Releases section says how to rebuild a release and compare it with PyPI. Measured before this change: two local builds of the same commit gave the same wheel and different sdists; after normalisation, the same sdist. Signed-off-by: arpan --- .github/workflows/ci.yml | 16 ++++++ .github/workflows/publish.yml | 5 ++ .github/workflows/release.yml | 2 + CONTRIBUTING.md | 17 ++++++ scripts/normalize_sdist.py | 74 +++++++++++++++++++++++++ tests/test_repository_signals.py | 93 ++++++++++++++++++++++++++++++++ 6 files changed, 207 insertions(+) create mode 100644 scripts/normalize_sdist.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ba4f417..3e06c0b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -398,10 +398,26 @@ jobs: with: python-version: "3.11" + # `SOURCE_DATE_EPOCH` is the commit's own timestamp, so a wheel built from this commit by + # anyone, anywhere, with the backend the lock names, is the same bytes. The sdist needs + # `scripts/normalize_sdist.py` for what setuptools records from the scratch directory + # (copy times, uid, filesystem order). The step after builds a second time and compares: + # "reproducible" is a claim until something checks it. - name: Build run: | pip install --require-hashes -r requirements/build.txt + export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" python -m build --no-isolation + python scripts/normalize_sdist.py dist/*.tar.gz + + - name: The build is reproducible + run: | + export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" + python -m build --no-isolation --outdir /tmp/again + python scripts/normalize_sdist.py /tmp/again/*.tar.gz + (cd dist && sha256sum ./*) > /tmp/first.sha256 + (cd /tmp/again && sha256sum ./*) | diff /tmp/first.sha256 - + echo "both builds produced:"; cat /tmp/first.sha256 - name: Metadata renders on PyPI run: twine check --strict dist/* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5b4f9a3a..bd7f80a8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,10 +51,15 @@ jobs: echo "kernel=$kernel" >> "$GITHUB_OUTPUT" echo "publishing $target from $path" + # Reproducible: `SOURCE_DATE_EPOCH` is the tagged commit's timestamp and the sdist is + # normalised, so what PyPI serves is what anyone rebuilds from the tag (CONTRIBUTING.md, + # Releases). CI's package job checks the property on every pull request. - name: Build run: | pip install --require-hashes -r requirements/build.txt + export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" python -m build --no-isolation --outdir dist "${{ steps.target.outputs.path }}" + python scripts/normalize_sdist.py dist/*.tar.gz - name: Metadata renders on PyPI run: twine check --strict dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40d704bf..7aece283 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,7 +77,9 @@ jobs: if: steps.existing.outputs.exists == 'false' run: | pip install --require-hashes -r requirements/build.txt + export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" python -m build --no-isolation + python scripts/normalize_sdist.py dist/*.tar.gz # Signed provenance for the artifacts this release is about to carry. The action signs # against the workflow's OIDC identity, so what it attests is *this* workflow, at *this* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 51802ce3..2b06bab6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -193,6 +193,23 @@ tracked. Adapters ship on their own version line (`adapters-langgraph-1.0`), from `adapters/`, and gate no kernel release. +**A release can be rebuilt by anyone and compared byte for byte.** The workflows set +`SOURCE_DATE_EPOCH` to the tagged commit's timestamp and normalise the sdist with +`scripts/normalize_sdist.py`, so from a clean clone of the tag, with the interpreter series +CI uses (3.11): + +```bash +pip install --require-hashes -r requirements/build.txt +export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" +python -m build --no-isolation +python scripts/normalize_sdist.py dist/*.tar.gz +sha256sum dist/* +``` + +The hashes match the distributions on PyPI and on the GitHub Release, for every release cut +after this was added. CI's `package` job builds every pull request twice and fails if the two +differ, so the property is checked before a tag rather than claimed after one. + ## Reporting a vulnerability Privately, per [SECURITY.md](SECURITY.md). If an action ran that policy should have refused, diff --git a/scripts/normalize_sdist.py b/scripts/normalize_sdist.py new file mode 100644 index 00000000..04c7b562 --- /dev/null +++ b/scripts/normalize_sdist.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2026 The CTRLRun contributors +# SPDX-License-Identifier: Apache-2.0 +"""Rewrite an sdist so that two builds of the same commit are the same bytes. + +`python -m build` writes the wheel reproducibly when `SOURCE_DATE_EPOCH` is set, and the sdist +not quite: setuptools copies the tree into a scratch directory and the tar records the copy +time, the uid, the gid and the order the filesystem returned. This script keeps every member +and every byte of content and fixes only what the build could not: members sorted by name, +every mtime set to `SOURCE_DATE_EPOCH`, owner `0:0` with empty names, permissions normalised to +`0o644` (files) or `0o755` (directories and files that were executable), and a gzip header with +no name and a zero timestamp. The result is a valid sdist that `pip` and `twine` read as before. + + SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) python scripts/normalize_sdist.py dist/*.tar.gz + +The wheel needs no help, and this script refuses one. +""" + +from __future__ import annotations + +import gzip +import io +import os +import sys +import tarfile +from pathlib import Path + + +def normalize(path: Path, epoch: int) -> None: + if not path.name.endswith(".tar.gz"): + raise SystemExit(f"normalize_sdist: {path} is not a .tar.gz sdist") + with tarfile.open(path, "r:gz") as source: + members = sorted(source.getmembers(), key=lambda m: m.name) + blobs: dict[str, bytes] = {} + for member in members: + if member.isfile(): + extracted = source.extractfile(member) + assert extracted is not None + blobs[member.name] = extracted.read() + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as target: + for member in members: + executable = member.isdir() or (member.mode & 0o111) + member.mtime = epoch + member.uid = member.gid = 0 + member.uname = member.gname = "" + member.mode = 0o755 if executable else 0o644 + member.pax_headers = {} + if member.isfile(): + target.addfile(member, io.BytesIO(blobs[member.name])) + else: + target.addfile(member) + with ( + open(path, "wb") as out, + gzip.GzipFile(filename="", mode="wb", fileobj=out, mtime=0) as zipped, + ): + zipped.write(raw.getvalue()) + + +def main(argv: list[str]) -> int: + if not argv: + print("usage: normalize_sdist.py ...", file=sys.stderr) + return 2 + epoch_text = os.environ.get("SOURCE_DATE_EPOCH") + if not epoch_text or not epoch_text.isdigit(): + print("normalize_sdist: SOURCE_DATE_EPOCH must be set to an integer", file=sys.stderr) + return 2 + for name in argv: + normalize(Path(name), int(epoch_text)) + print(f"normalized {name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/test_repository_signals.py b/tests/test_repository_signals.py index 665973a8..f5937bc8 100644 --- a/tests/test_repository_signals.py +++ b/tests/test_repository_signals.py @@ -10,10 +10,13 @@ import base64 import hashlib +import io import json +import os import re import subprocess import sys +import tarfile import tomllib from pathlib import Path @@ -607,3 +610,93 @@ def test_every_lock_a_workflow_installs_from_exists_and_is_hashed(): for index, line in enumerate(lines): if line and not line.startswith(("#", " ")): assert "--hash=" in lines[index + 1], f"{lock}: {line} carries no hash" + + +# --- a build anyone can repeat --------------------------------------------------------------- + + +def test_every_build_in_a_workflow_is_reproducible(): + """Every `python -m build` in a workflow runs with `SOURCE_DATE_EPOCH` set to the commit's + timestamp and normalises the sdist afterwards, so the distributions a tag publishes are the + ones a reader rebuilds from it (CONTRIBUTING.md, Releases).""" + found = 0 + for path in sorted(WORKFLOWS.glob("*.yml")): + workflow = yaml.safe_load(path.read_text(encoding="utf-8")) + for job in workflow["jobs"].values(): + for step in job.get("steps", []): + run = str(step.get("run", "")) + if "python -m build" not in run: + continue + found += 1 + lines = [line.strip() for line in run.splitlines() if line.strip()] + build = next(i for i, line in enumerate(lines) if "python -m build" in line) + assert 'export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)"' in lines[:build], ( + f"{path.name}: {step.get('name')} builds without SOURCE_DATE_EPOCH" + ) + assert any("scripts/normalize_sdist.py" in line for line in lines[build:]), ( + f"{path.name}: {step.get('name')} builds without normalising the sdist" + ) + assert found >= 4, found # ci.yml twice, publish.yml, release.yml + + +def _tarball(path: Path, files: dict[str, bytes], *, mtime: int, uid: int, order: list[str]): + with tarfile.open(path, "w:gz") as tar: + for name in order: + info = tarfile.TarInfo(name) + info.size = len(files[name]) + info.mtime = mtime + info.uid = info.gid = uid + info.uname = info.gname = "somebody" + info.mode = 0o664 + tar.addfile(info, io.BytesIO(files[name])) + + +def test_normalize_sdist_makes_two_builds_of_the_same_tree_identical(tmp_path): + files = {"pkg-1.0/PKG-INFO": b"Name: pkg\n", "pkg-1.0/src/a.py": b"print(1)\n"} + first, second = tmp_path / "first.tar.gz", tmp_path / "second.tar.gz" + _tarball(first, files, mtime=1_700_000_000, uid=1000, order=list(files)) + _tarball(second, files, mtime=1_700_000_099, uid=1001, order=list(reversed(files))) + assert first.read_bytes() != second.read_bytes() + + result = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "normalize_sdist.py"), + str(first), + str(second), + ], + capture_output=True, + text=True, + env={**os.environ, "SOURCE_DATE_EPOCH": "1789312180"}, + ) + assert result.returncode == 0, result.stderr + assert first.read_bytes() == second.read_bytes() + with tarfile.open(first, "r:gz") as tar: + members = tar.getmembers() + assert [m.name for m in members] == sorted(files) + assert {m.mtime for m in members} == {1789312180} + assert {(m.uid, m.gid, m.uname, m.gname, m.mode) for m in members} == { + (0, 0, "", "", 0o644) + } + for member in members: + extracted = tar.extractfile(member) + assert extracted is not None and extracted.read() == files[member.name] + + +def test_normalize_sdist_refuses_without_an_epoch_and_refuses_a_wheel(tmp_path): + wheel = tmp_path / "pkg-1.0-py3-none-any.whl" + wheel.write_bytes(b"not a tar") + without = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "normalize_sdist.py"), str(wheel)], + capture_output=True, + text=True, + env={k: v for k, v in os.environ.items() if k != "SOURCE_DATE_EPOCH"}, + ) + assert without.returncode == 2 and "SOURCE_DATE_EPOCH" in without.stderr + refused = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "normalize_sdist.py"), str(wheel)], + capture_output=True, + text=True, + env={**os.environ, "SOURCE_DATE_EPOCH": "1"}, + ) + assert refused.returncode != 0 and "not a .tar.gz sdist" in refused.stderr