From 0f7c65a230b0411571ef36decb4343f991cbb8f8 Mon Sep 17 00:00:00 2001 From: Dreven Date: Sat, 24 Oct 2026 17:21:41 +0300 Subject: [PATCH] ci: verify isolated installed wheel and reject broken distributions (#84) --- .github/workflows/ci.yml | 14 ++++ CONTRIBUTING.md | 4 + docs/wheel-verification.md | 33 ++++++++ scripts/wheel_smoke.py | 163 +++++++++++++++++++++++++++++++++++++ tests/test_wheel_smoke.py | 66 +++++++++++++++ 5 files changed, 280 insertions(+) create mode 100644 docs/wheel-verification.md create mode 100644 scripts/wheel_smoke.py create mode 100644 tests/test_wheel_smoke.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c94bd7f..ddb3739 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,20 @@ permissions: contents: read jobs: + wheel-smoke: + name: Installed wheel (Ubuntu, Python 3.12) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build and verify the installed wheel outside checkout + run: python scripts/wheel_smoke.py + test: name: Tests (Python ${{ matrix.python }}) runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6e1c21..a697ac9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,6 +98,10 @@ adapter against a repository you can read with ## Pull request checklist +For distribution changes, also run the [installed-wheel gate](docs/wheel-verification.md). +It installs the actual wheel in a clean environment outside the source checkout; +the editable source-test run above covers a different boundary. + - the description carries `Target: ` when the change advances a declared target, and the id exists in `.devostasis/targets.json`; - tests pass on Python 3.12 or newer, and no new dependency was added to the diff --git a/docs/wheel-verification.md b/docs/wheel-verification.md new file mode 100644 index 0000000..61e33c7 --- /dev/null +++ b/docs/wheel-verification.md @@ -0,0 +1,33 @@ +# Installed-wheel verification + +Source tests and an editable installation do not prove that the distribution +contains a working CLI. From a Python 3.12+ checkout with pip available, run: + +```bash +python scripts/wheel_smoke.py +``` + +The gate builds one wheel from this checkout; build isolation can download the +declared setuptools build dependency. It checks the source/metadata version, +required package members and console entry point, and refuses dependencies that +are not provably optional extras. Complex dependency markers fail closed. + +It installs only that wheel with `--no-index --no-deps` in a fresh temporary venv, +removes inherited Python import paths, and runs outside the checkout. All example +inputs are copied there. Isolated import/version checks, CLI help/version, core, +work and code example verification/replay, and an offline fixture build must pass. +Four corrupted copies of the built wheel must fail admission: missing CLI, wrong +version, runtime dependency, and an optional-extra OR bypass. No corrupted wheel +is installed. The installed runtime smoke performs no network collection. + +Every subprocess has a timeout; CI additionally limits the job to ten minutes. +Scratch files are removed on exit. Use `--scratch-root ` to +select a temporary-files location outside the checkout. A successful run prints +JSON with the wheel, environment and checks; any failed command exits nonzero. + +The separate CI job targets **Ubuntu/Python 3.12** at the exact PR head (or push +SHA), alongside the existing source-test matrix. Windows can run the same script; +only a recorded successful execution qualifies that version/platform. macOS, +other wheel-consumer combinations, privileged path behavior, PyPI publication +and release acceptance remain unqualified by this gate. The current 0.6.0 +checkout is an unreleased candidate; this command does not publish it. diff --git a/scripts/wheel_smoke.py b/scripts/wheel_smoke.py new file mode 100644 index 0000000..ad7b5f6 --- /dev/null +++ b/scripts/wheel_smoke.py @@ -0,0 +1,163 @@ +"""Build and exercise the distribution without importing the source checkout.""" + +from __future__ import annotations + +import argparse +import configparser +from email.parser import BytesParser +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import zipfile + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def inspect_wheel(wheel: Path, expected_version: str) -> None: + """Refuse incomplete/version-drifted wheels and unproven runtime dependencies.""" + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + require(len(names) == len(set(names)), "duplicate wheel members") + metadata_paths = [p for p in names if p.endswith(".dist-info/METADATA")] + require(len(metadata_paths) == 1, "expected one distribution METADATA") + metadata_path = metadata_paths[0] + info = metadata_path.rsplit("/", 1)[0] + metadata = BytesParser().parsebytes(archive.read(metadata_path)) + require(metadata.get("Name", "").lower() == "devostasis", "wrong distribution name") + require(metadata.get_all("Version") == [expected_version], "wheel/source version mismatch") + for requirement in metadata.get_all("Requires-Dist", []): + distribution, separator, marker = requirement.partition(";") + # Only the simple optional-extra form emitted by this project is + # proven safe. An `or python_version ...` must not hide a runtime dep. + require(bool(separator and distribution.strip() and re.fullmatch( + r"\s*extra\s*==\s*(?:\"[A-Za-z0-9_.-]+\"|'[A-Za-z0-9_.-]+')\s*", marker + )), f"dependency is not provably extra-only: {requirement}") + for member in ("devostasis/__init__.py", "devostasis/cli.py", "devostasis/__main__.py", + f"{info}/WHEEL", f"{info}/RECORD", f"{info}/entry_points.txt"): + require(member in names, f"missing wheel member: {member}") + entry_points = configparser.ConfigParser(interpolation=None) + entry_points.read_string(archive.read(f"{info}/entry_points.txt").decode("utf-8")) + require(entry_points.get("console_scripts", "devostasis", fallback="") == "devostasis.cli:main", + "missing or incorrect console entry point") + + +def negative_controls(wheel: Path, expected_version: str, directory: Path) -> list[str]: + """Corrupt the actual built wheel; source imports must never rescue it.""" + cases = [] + with zipfile.ZipFile(wheel) as source: + members = {p: source.read(p) for p in source.namelist()} + metadata_path = next(p for p in members if p.endswith(".dist-info/METADATA")) + for label in ("missing-cli", "wrong-version", "runtime-dependency", "extra-or-bypass"): + damaged = dict(members) + if label == "missing-cli": + del damaged["devostasis/cli.py"] + elif label == "wrong-version": + damaged[metadata_path] = re.sub( + rb"(?m)^Version: [^\r\n]+", b"Version: 0.0.0+invalid", damaged[metadata_path] + ) + else: + dependency = ("Requires-Dist: wheel-smoke-unexpected\n" if label == "runtime-dependency" + else 'Requires-Dist: wheel-smoke-unexpected; extra == "dev" or python_version < "99"\n') + # Insert a new header before the metadata body. + damaged[metadata_path] = dependency.encode() + damaged[metadata_path] + path = directory / f"{label}.whl" + with zipfile.ZipFile(path, "w") as archive: + for name, data in damaged.items(): + archive.writestr(name, data) + try: + inspect_wheel(path, expected_version) + except ValueError: + cases.append(label) + else: + raise ValueError(f"negative control unexpectedly accepted: {label}") + return cases + + +def run(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int = 120) -> str: + result = subprocess.run(command, cwd=cwd, env=env, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + if result.returncode: + raise RuntimeError(f"command failed ({result.returncode}): {command}\n{result.stdout}\n{result.stderr}") + return result.stdout + + +def smoke(source: Path, scratch_root: Path | None = None) -> dict[str, object]: + project = tomllib.loads((source / "pyproject.toml").read_text("utf-8"))["project"] + version = project["version"] + require(project.get("dependencies") == [], "source declares runtime dependencies") + env = {k: v for k, v in os.environ.items() if k.upper() not in ("PYTHONPATH", "PYTHONHOME")} + env["PYTHONDONTWRITEBYTECODE"] = "1" + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + with tempfile.TemporaryDirectory(prefix="devostasis-wheel-", dir=scratch_root) as temporary: + root = Path(temporary).resolve() + require(not root.is_relative_to(source), "smoke directory must be outside the checkout") + wheels = root / "dist" + run([sys.executable, "-I", "-m", "pip", "wheel", "--no-deps", "--wheel-dir", str(wheels), str(source)], + cwd=root, env=env, timeout=240) + built = list(wheels.glob("*.whl")) + require(len(built) == 1, "expected exactly one built wheel") + inspect_wheel(built[0], version) + controls = negative_controls(built[0], version, root) + venv = root / "venv" + run([sys.executable, "-I", "-m", "venv", str(venv)], cwd=root, env=env) + executable_dir = venv / ("Scripts" if os.name == "nt" else "bin") + python = executable_dir / ("python.exe" if os.name == "nt" else "python") + cli = executable_dir / ("devostasis.exe" if os.name == "nt" else "devostasis") + run([str(python), "-I", "-m", "pip", "install", "--no-index", "--no-deps", str(built[0])], cwd=root, env=env) + outside = root / "outside" + outside.mkdir() + fixtures = outside / "fixtures" + fixtures.mkdir() + shutil.copy2(source / "examples/observations.json", fixtures / "observations.json") + for name in ("sample-bundle", "work/bundle", "code/bundle"): + shutil.copytree(source / "examples" / name, fixtures / name) + shutil.copy2(source / "examples/code/packet.json", fixtures / "code/packet.json") + # Inspect the actual isolated interpreter, not a source-tree import. + check = """import importlib.metadata as m, json, pathlib, sys, devostasis +d = m.distribution('devostasis') +assert d.version == sys.argv[1] == devostasis.__version__ +assert pathlib.Path(devostasis.__file__).resolve().is_relative_to(pathlib.Path(sys.prefix).resolve()) +print(json.dumps({'version': d.version, 'import': 'fresh-venv'})) +""" + installed = json.loads(run([str(python), "-I", "-c", check, version], cwd=outside, env=env)) + require(run([str(cli), "--version"], cwd=outside, env=env).strip() == f"devostasis {version}", "CLI version mismatch") + run([str(cli), "--help"], cwd=outside, env=env) + commands = [ + ["verify", "--bundle", str(fixtures / "sample-bundle")], + ["work", "verify", "--bundle", str(fixtures / "work/bundle")], + ["work", "replay", "--bundle", str(fixtures / "work/bundle")], + ["code", "verify", "--bundle", str(fixtures / "code/bundle")], + ["code", "verify-packet", "--bundle", str(fixtures / "code/bundle"), "--packet", str(fixtures / "code/packet.json")], + ["build", "--observations", str(fixtures / "observations.json"), "--store", str(outside / "store"), + "--planning", "milestones", "--debt-label", "type:refactor", "--debt-mapping-version", "example-1"], + ["verify", "--bundle", str(outside / "store/projects/github.com/acme/widget/latest")], + ] + for command in commands: + run([str(cli), *command], cwd=outside, env=env) + return {"wheel": built[0].name, "installed": installed, "negative_controls": controls, + "offline_commands": len(commands), "platform": sys.platform, "python": sys.version.split()[0]} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scratch-root", type=Path, help="existing directory outside the source checkout") + args = parser.parse_args() + try: + print(json.dumps(smoke(Path(__file__).resolve().parents[1], args.scratch_root), indent=2)) + except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired, zipfile.BadZipFile) as exc: + print(f"wheel smoke failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_wheel_smoke.py b/tests/test_wheel_smoke.py new file mode 100644 index 0000000..75c4a63 --- /dev/null +++ b/tests/test_wheel_smoke.py @@ -0,0 +1,66 @@ +"""Packaging admission must reject wheels that source-tree tests could conceal.""" + +import importlib.util +from pathlib import Path +import zipfile + +import pytest + +spec = importlib.util.spec_from_file_location("wheel_smoke", Path(__file__).parents[1] / "scripts/wheel_smoke.py") +wheel_smoke = importlib.util.module_from_spec(spec) +spec.loader.exec_module(wheel_smoke) + + +def make_wheel(tmp_path, *, dependency=None, entry="devostasis.cli:main", missing=None, version="1.2.3"): + info = "devostasis-1.2.3.dist-info" + metadata = f"Metadata-Version: 2.4\nName: devostasis\nVersion: {version}\n" + if dependency: + metadata += f"Requires-Dist: {dependency}\n" + members = {f"{info}/METADATA": metadata, f"{info}/WHEEL": "Wheel-Version: 1.0\n", + f"{info}/RECORD": "", f"{info}/entry_points.txt": f"[console_scripts]\ndevostasis = {entry}\n", + "devostasis/__init__.py": "", "devostasis/__main__.py": "", "devostasis/cli.py": ""} + if missing: + del members[missing] + wheel = tmp_path / "devostasis-1.2.3-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for name, data in members.items(): + archive.writestr(name, data) + return wheel + + +@pytest.mark.parametrize("dependency", [None, 'pytest>=8; extra == "dev"', "pytest>=8; extra == 'dev'"]) +def test_accepts_empty_runtime_dependencies_and_simple_dev_extra(tmp_path, dependency): + wheel_smoke.inspect_wheel(make_wheel(tmp_path, dependency=dependency), "1.2.3") + + +@pytest.mark.parametrize("dependency", [ + "requests>=2", 'requests; python_version < "99"', + 'requests; extra == "dev" or python_version < "99"', + 'requests; extra == "dev" and python_version < "99"', 'requests; extra != "dev"', +]) +def test_refuses_runtime_or_unproven_complex_dependencies(tmp_path, dependency): + with pytest.raises(ValueError, match="not provably extra-only"): + wheel_smoke.inspect_wheel(make_wheel(tmp_path, dependency=dependency), "1.2.3") + + +def test_refuses_wrong_version(tmp_path): + with pytest.raises(ValueError, match="version mismatch"): + wheel_smoke.inspect_wheel(make_wheel(tmp_path, version="9.9.9"), "1.2.3") + + +def test_refuses_missing_cli(tmp_path): + with pytest.raises(ValueError, match="missing wheel member"): + wheel_smoke.inspect_wheel(make_wheel(tmp_path, missing="devostasis/cli.py"), "1.2.3") + + +def test_refuses_wrong_entry_point(tmp_path): + with pytest.raises(ValueError, match="console entry point"): + wheel_smoke.inspect_wheel(make_wheel(tmp_path, entry="unrelated:main"), "1.2.3") + + +def test_negative_controls_damage_the_actual_wheel(tmp_path): + wheel = make_wheel(tmp_path) + assert wheel_smoke.negative_controls(wheel, "1.2.3", tmp_path) == [ + "missing-cli", "wrong-version", "runtime-dependency", "extra-or-bypass", + ] + wheel_smoke.inspect_wheel(wheel, "1.2.3")