ci: verify installed wheel outside the source checkout (#84)

.github/workflows/ci.yml

@@ -14,6 +14,20 @@ permissions:
   contents: read
 
 jobs:
+  wheel-smoke:
+    name: Installed wheel (Ubuntu, Python 3.12)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build and verify the installed wheel outside checkout
+        run: python scripts/wheel_smoke.py
+
   test:
     name: Tests (Python ${{ matrix.python }})
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -98,6 +98,10 @@ adapter against a repository you can read with
 
 ## Pull request checklist
 
+For distribution changes, also run the [installed-wheel gate](docs/wheel-verification.md).
+It installs the actual wheel in a clean environment outside the source checkout;
+the editable source-test run above covers a different boundary.
+
 - the description carries `Target: <id>` when the change advances a declared
   target, and the id exists in `.devostasis/targets.json`;
 - tests pass on Python 3.12 or newer, and no new dependency was added to the

docs/wheel-verification.md

@@ -0,0 +1,33 @@
+# Installed-wheel verification
+
+Source tests and an editable installation do not prove that the distribution
+contains a working CLI. From a Python 3.12+ checkout with pip available, run:
+
+```bash
+python scripts/wheel_smoke.py
+```
+
+The gate builds one wheel from this checkout; build isolation can download the
+declared setuptools build dependency. It checks the source/metadata version,
+required package members and console entry point, and refuses dependencies that
+are not provably optional extras. Complex dependency markers fail closed.
+
+It installs only that wheel with `--no-index --no-deps` in a fresh temporary venv,
+removes inherited Python import paths, and runs outside the checkout. All example
+inputs are copied there. Isolated import/version checks, CLI help/version, core,
+work and code example verification/replay, and an offline fixture build must pass.
+Four corrupted copies of the built wheel must fail admission: missing CLI, wrong
+version, runtime dependency, and an optional-extra OR bypass. No corrupted wheel
+is installed. The installed runtime smoke performs no network collection.
+
+Every subprocess has a timeout; CI additionally limits the job to ten minutes.
+Scratch files are removed on exit. Use `--scratch-root <existing-directory>` to
+select a temporary-files location outside the checkout. A successful run prints
+JSON with the wheel, environment and checks; any failed command exits nonzero.
+
+The separate CI job targets **Ubuntu/Python 3.12** at the exact PR head (or push
+SHA), alongside the existing source-test matrix. Windows can run the same script;
+only a recorded successful execution qualifies that version/platform. macOS,
+other wheel-consumer combinations, privileged path behavior, PyPI publication
+and release acceptance remain unqualified by this gate. The current 0.6.0
+checkout is an unreleased candidate; this command does not publish it.

scripts/wheel_smoke.py

@@ -0,0 +1,163 @@
+"""Build and exercise the distribution without importing the source checkout."""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+from email.parser import BytesParser
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import tomllib
+import zipfile
+
+
+def require(condition: bool, message: str) -> None:
+    if not condition:
+        raise ValueError(message)
+
+
+def inspect_wheel(wheel: Path, expected_version: str) -> None:
+    """Refuse incomplete/version-drifted wheels and unproven runtime dependencies."""
+    with zipfile.ZipFile(wheel) as archive:
+        names = archive.namelist()
+        require(len(names) == len(set(names)), "duplicate wheel members")
+        metadata_paths = [p for p in names if p.endswith(".dist-info/METADATA")]
+        require(len(metadata_paths) == 1, "expected one distribution METADATA")
+        metadata_path = metadata_paths[0]
+        info = metadata_path.rsplit("/", 1)[0]
+        metadata = BytesParser().parsebytes(archive.read(metadata_path))
+        require(metadata.get("Name", "").lower() == "devostasis", "wrong distribution name")
+        require(metadata.get_all("Version") == [expected_version], "wheel/source version mismatch")
+        for requirement in metadata.get_all("Requires-Dist", []):
+            distribution, separator, marker = requirement.partition(";")
+            # Only the simple optional-extra form emitted by this project is
+            # proven safe. An `or python_version ...` must not hide a runtime dep.
+            require(bool(separator and distribution.strip() and re.fullmatch(
+                r"\s*extra\s*==\s*(?:\"[A-Za-z0-9_.-]+\"|'[A-Za-z0-9_.-]+')\s*", marker
+            )), f"dependency is not provably extra-only: {requirement}")
+        for member in ("devostasis/__init__.py", "devostasis/cli.py", "devostasis/__main__.py",
+                       f"{info}/WHEEL", f"{info}/RECORD", f"{info}/entry_points.txt"):
+            require(member in names, f"missing wheel member: {member}")
+        entry_points = configparser.ConfigParser(interpolation=None)
+        entry_points.read_string(archive.read(f"{info}/entry_points.txt").decode("utf-8"))
+        require(entry_points.get("console_scripts", "devostasis", fallback="") == "devostasis.cli:main",
+                "missing or incorrect console entry point")
+
+
+def negative_controls(wheel: Path, expected_version: str, directory: Path) -> list[str]:
+    """Corrupt the actual built wheel; source imports must never rescue it."""
+    cases = []
+    with zipfile.ZipFile(wheel) as source:
+        members = {p: source.read(p) for p in source.namelist()}
+    metadata_path = next(p for p in members if p.endswith(".dist-info/METADATA"))
+    for label in ("missing-cli", "wrong-version", "runtime-dependency", "extra-or-bypass"):
+        damaged = dict(members)
+        if label == "missing-cli":
+            del damaged["devostasis/cli.py"]
+        elif label == "wrong-version":
+            damaged[metadata_path] = re.sub(
+                rb"(?m)^Version: [^\r\n]+", b"Version: 0.0.0+invalid", damaged[metadata_path]
+            )
+        else:
+            dependency = ("Requires-Dist: wheel-smoke-unexpected\n" if label == "runtime-dependency"
+                          else 'Requires-Dist: wheel-smoke-unexpected; extra == "dev" or python_version < "99"\n')
+            # Insert a new header before the metadata body.
+            damaged[metadata_path] = dependency.encode() + damaged[metadata_path]
+        path = directory / f"{label}.whl"
+        with zipfile.ZipFile(path, "w") as archive:
+            for name, data in damaged.items():
+                archive.writestr(name, data)
+        try:
+            inspect_wheel(path, expected_version)
+        except ValueError:
+            cases.append(label)
+        else:
+            raise ValueError(f"negative control unexpectedly accepted: {label}")
+    return cases
+
+
+def run(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int = 120) -> str:
+    result = subprocess.run(command, cwd=cwd, env=env, capture_output=True, text=True,
+                            encoding="utf-8", errors="replace", timeout=timeout)
+    if result.returncode:
+        raise RuntimeError(f"command failed ({result.returncode}): {command}\n{result.stdout}\n{result.stderr}")
+    return result.stdout
+
+
+def smoke(source: Path, scratch_root: Path | None = None) -> dict[str, object]:
+    project = tomllib.loads((source / "pyproject.toml").read_text("utf-8"))["project"]
+    version = project["version"]
+    require(project.get("dependencies") == [], "source declares runtime dependencies")
+    env = {k: v for k, v in os.environ.items() if k.upper() not in ("PYTHONPATH", "PYTHONHOME")}
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
+    env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+    with tempfile.TemporaryDirectory(prefix="devostasis-wheel-", dir=scratch_root) as temporary:
+        root = Path(temporary).resolve()
+        require(not root.is_relative_to(source), "smoke directory must be outside the checkout")
+        wheels = root / "dist"
+        run([sys.executable, "-I", "-m", "pip", "wheel", "--no-deps", "--wheel-dir", str(wheels), str(source)],
+            cwd=root, env=env, timeout=240)
+        built = list(wheels.glob("*.whl"))
+        require(len(built) == 1, "expected exactly one built wheel")
+        inspect_wheel(built[0], version)
+        controls = negative_controls(built[0], version, root)
+        venv = root / "venv"
+        run([sys.executable, "-I", "-m", "venv", str(venv)], cwd=root, env=env)
+        executable_dir = venv / ("Scripts" if os.name == "nt" else "bin")
+        python = executable_dir / ("python.exe" if os.name == "nt" else "python")
+        cli = executable_dir / ("devostasis.exe" if os.name == "nt" else "devostasis")
+        run([str(python), "-I", "-m", "pip", "install", "--no-index", "--no-deps", str(built[0])], cwd=root, env=env)
+        outside = root / "outside"
+        outside.mkdir()
+        fixtures = outside / "fixtures"
+        fixtures.mkdir()
+        shutil.copy2(source / "examples/observations.json", fixtures / "observations.json")
+        for name in ("sample-bundle", "work/bundle", "code/bundle"):
+            shutil.copytree(source / "examples" / name, fixtures / name)
+        shutil.copy2(source / "examples/code/packet.json", fixtures / "code/packet.json")
+        # Inspect the actual isolated interpreter, not a source-tree import.
+        check = """import importlib.metadata as m, json, pathlib, sys, devostasis
+d = m.distribution('devostasis')
+assert d.version == sys.argv[1] == devostasis.__version__
+assert pathlib.Path(devostasis.__file__).resolve().is_relative_to(pathlib.Path(sys.prefix).resolve())
+print(json.dumps({'version': d.version, 'import': 'fresh-venv'}))
+"""
+        installed = json.loads(run([str(python), "-I", "-c", check, version], cwd=outside, env=env))
+        require(run([str(cli), "--version"], cwd=outside, env=env).strip() == f"devostasis {version}", "CLI version mismatch")
+        run([str(cli), "--help"], cwd=outside, env=env)
+        commands = [
+            ["verify", "--bundle", str(fixtures / "sample-bundle")],
+            ["work", "verify", "--bundle", str(fixtures / "work/bundle")],
+            ["work", "replay", "--bundle", str(fixtures / "work/bundle")],
+            ["code", "verify", "--bundle", str(fixtures / "code/bundle")],
+            ["code", "verify-packet", "--bundle", str(fixtures / "code/bundle"), "--packet", str(fixtures / "code/packet.json")],
+            ["build", "--observations", str(fixtures / "observations.json"), "--store", str(outside / "store"),
+             "--planning", "milestones", "--debt-label", "type:refactor", "--debt-mapping-version", "example-1"],
+            ["verify", "--bundle", str(outside / "store/projects/github.com/acme/widget/latest")],
+        ]
+        for command in commands:
+            run([str(cli), *command], cwd=outside, env=env)
+        return {"wheel": built[0].name, "installed": installed, "negative_controls": controls,
+                "offline_commands": len(commands), "platform": sys.platform, "python": sys.version.split()[0]}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--scratch-root", type=Path, help="existing directory outside the source checkout")
+    args = parser.parse_args()
+    try:
+        print(json.dumps(smoke(Path(__file__).resolve().parents[1], args.scratch_root), indent=2))
+    except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired, zipfile.BadZipFile) as exc:
+        print(f"wheel smoke failed: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_wheel_smoke.py

@@ -0,0 +1,66 @@
+"""Packaging admission must reject wheels that source-tree tests could conceal."""
+
+import importlib.util
+from pathlib import Path
+import zipfile
+
+import pytest
+
+spec = importlib.util.spec_from_file_location("wheel_smoke", Path(__file__).parents[1] / "scripts/wheel_smoke.py")
+wheel_smoke = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(wheel_smoke)
+
+
+def make_wheel(tmp_path, *, dependency=None, entry="devostasis.cli:main", missing=None, version="1.2.3"):
+    info = "devostasis-1.2.3.dist-info"
+    metadata = f"Metadata-Version: 2.4\nName: devostasis\nVersion: {version}\n"
+    if dependency:
+        metadata += f"Requires-Dist: {dependency}\n"
+    members = {f"{info}/METADATA": metadata, f"{info}/WHEEL": "Wheel-Version: 1.0\n",
+               f"{info}/RECORD": "", f"{info}/entry_points.txt": f"[console_scripts]\ndevostasis = {entry}\n",
+               "devostasis/__init__.py": "", "devostasis/__main__.py": "", "devostasis/cli.py": ""}
+    if missing:
+        del members[missing]
+    wheel = tmp_path / "devostasis-1.2.3-py3-none-any.whl"
+    with zipfile.ZipFile(wheel, "w") as archive:
+        for name, data in members.items():
+            archive.writestr(name, data)
+    return wheel
+
+
+@pytest.mark.parametrize("dependency", [None, 'pytest>=8; extra == "dev"', "pytest>=8; extra == 'dev'"])
+def test_accepts_empty_runtime_dependencies_and_simple_dev_extra(tmp_path, dependency):
+    wheel_smoke.inspect_wheel(make_wheel(tmp_path, dependency=dependency), "1.2.3")
+
+
+@pytest.mark.parametrize("dependency", [
+    "requests>=2", 'requests; python_version < "99"',
+    'requests; extra == "dev" or python_version < "99"',
+    'requests; extra == "dev" and python_version < "99"', 'requests; extra != "dev"',
+])
+def test_refuses_runtime_or_unproven_complex_dependencies(tmp_path, dependency):
+    with pytest.raises(ValueError, match="not provably extra-only"):
+        wheel_smoke.inspect_wheel(make_wheel(tmp_path, dependency=dependency), "1.2.3")
+
+
+def test_refuses_wrong_version(tmp_path):
+    with pytest.raises(ValueError, match="version mismatch"):
+        wheel_smoke.inspect_wheel(make_wheel(tmp_path, version="9.9.9"), "1.2.3")
+
+
+def test_refuses_missing_cli(tmp_path):
+    with pytest.raises(ValueError, match="missing wheel member"):
+        wheel_smoke.inspect_wheel(make_wheel(tmp_path, missing="devostasis/cli.py"), "1.2.3")
+
+
+def test_refuses_wrong_entry_point(tmp_path):
+    with pytest.raises(ValueError, match="console entry point"):
+        wheel_smoke.inspect_wheel(make_wheel(tmp_path, entry="unrelated:main"), "1.2.3")
+
+
+def test_negative_controls_damage_the_actual_wheel(tmp_path):
+    wheel = make_wheel(tmp_path)
+    assert wheel_smoke.negative_controls(wheel, "1.2.3", tmp_path) == [
+        "missing-cli", "wrong-version", "runtime-dependency", "extra-or-bypass",
+    ]
+    wheel_smoke.inspect_wheel(wheel, "1.2.3")