From 6a708ffaa1c9348db4f4c46664ee19826a410284 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Mon, 31 Aug 2026 20:39:22 +0200 Subject: [PATCH] ingest: accept dependency-free tagged handoffs --- scripts/tagged_release_handoff.py | 25 +++++-- tests/test_pfb_pkg.py | 61 ++++++++++++++++- tests/test_publish_release.py | 106 ++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 7 deletions(-) diff --git a/scripts/tagged_release_handoff.py b/scripts/tagged_release_handoff.py index c43e0b3..4160999 100755 --- a/scripts/tagged_release_handoff.py +++ b/scripts/tagged_release_handoff.py @@ -160,7 +160,7 @@ def _dependency_packages( *, ports_sha: str, source_date_epoch: int, - dependency_builder: Mapping[str, str], + dependency_builder: Mapping[str, str] | None, ) -> dict[str, dict[str, dict[str, object]]]: if not isinstance(value, Mapping): raise HandoffError("dependency_packages must be an object") @@ -271,10 +271,17 @@ def _validate_handoff_fields( rows = _route_matrix(route_matrix) if type(source_date_epoch) is not int or source_date_epoch < 0: raise HandoffError("source_date_epoch must be a non-negative integer") - try: - normalized_builder = validate_dependency_builder(dependency_builder) - except PkgError as exc: - raise HandoffError(str(exc)) from exc + if dependency_builder is None: + if any(row["extra_pkgs"] for row in rows): + raise HandoffError( + "dependency_builder is required when route_matrix contains extra packages" + ) + normalized_builder = None + else: + try: + normalized_builder = validate_dependency_builder(dependency_builder) + except PkgError as exc: + raise HandoffError(str(exc)) from exc normalized_packages = _dependency_packages( dependency_packages, rows, @@ -366,7 +373,6 @@ def validate_build_records( "source_sha": handoff.get("source_sha"), "freebsd_ports_sha": handoff.get("ports_sha"), "source_date_epoch": handoff.get("source_date_epoch"), - "dependency_builder": handoff.get("dependency_builder"), } for index, record in enumerate(records): if not isinstance(record, Mapping): @@ -374,6 +380,13 @@ def validate_build_records( for name, value in expected.items(): if record.get(name) != value: raise BuildRecordIdentityError(index, name) + dependency_builder = record.get("dependency_builder") + if dependency_builder is None: + row = record.get("matrix_row") + if not isinstance(row, Mapping) or row.get("extra_pkgs") != []: + raise BuildRecordIdentityError(index, "dependency_builder") + elif dependency_builder != handoff.get("dependency_builder"): + raise BuildRecordIdentityError(index, "dependency_builder") def _dependency_requirements( diff --git a/tests/test_pfb_pkg.py b/tests/test_pfb_pkg.py index c717aa5..b12793d 100644 --- a/tests/test_pfb_pkg.py +++ b/tests/test_pfb_pkg.py @@ -23,6 +23,16 @@ import pfb_pkg import pytest +DEPENDENCY_BUILDER = { + "python": "3.11.15", + "pip": "26.2.1", + "setuptools": "75.6.0", + "wheel": "0.45.1", + "zstandard": "0.25.0", + "uv": "0.12.6", + "uv_lock_sha256": "d" * 64, +} + def _record(**overrides: object) -> dict: row = { @@ -48,9 +58,10 @@ def _record(**overrides: object) -> dict: "native_recipe_identity": "pfSense-pkg-pfBlockerNG", "emitted_identity": "pfSense-pkg-pfBlockerNG", "matrix_row": row, - "freebsd_ports_sha": "b" * 64, + "freebsd_ports_sha": "b" * 40, "route": "stable/ce-2.8", "source_date_epoch": 0, + "dependency_builder": DEPENDENCY_BUILDER, "build_input_digest": "", } record.update(overrides) @@ -68,6 +79,48 @@ def test_build_record_valid_and_digest_is_canonical() -> None: assert pfb_pkg.build_input_digest(changed) != record["build_input_digest"] +def test_build_record_allows_missing_dependency_builder_without_extra_packages() -> None: + record = _record() + record["matrix_row"] = {**record["matrix_row"], "extra_pkgs": []} + record.pop("dependency_builder") + record["build_input_digest"] = pfb_pkg.build_input_digest(record) + + assert pfb_pkg.validate_build_record(record) == record + + +def test_build_record_allows_valid_dependency_builder_without_extra_packages() -> None: + record = _record() + record["matrix_row"] = {**record["matrix_row"], "extra_pkgs": []} + record["build_input_digest"] = pfb_pkg.build_input_digest(record) + + assert pfb_pkg.validate_build_record(record) == record + + +def test_build_record_rejects_malformed_dependency_builder_without_extra_packages() -> None: + record = _record(dependency_builder={**DEPENDENCY_BUILDER, "wheel": "not-a-version"}) + record["matrix_row"] = {**record["matrix_row"], "extra_pkgs": []} + record["build_input_digest"] = pfb_pkg.build_input_digest(record) + + with pytest.raises(pfb_pkg.PkgError, match="dependency_builder"): + pfb_pkg.validate_build_record(record) + + +def test_build_record_allows_legacy_missing_builder_with_extra_packages() -> None: + record = _record(freebsd_ports_sha="b" * 64) + record.pop("dependency_builder") + record["build_input_digest"] = pfb_pkg.build_input_digest(record) + + assert pfb_pkg.validate_build_record(record) == record + + +def test_build_record_rejects_malformed_dependency_builder() -> None: + record = _record(dependency_builder={**DEPENDENCY_BUILDER, "wheel": "not-a-version"}) + with pytest.raises(pfb_pkg.PkgError, match="dependency_builder"): + pfb_pkg.validate_build_record(record) + with pytest.raises(pfb_pkg.PkgError, match="dependency_builder"): + pfb_pkg.load_build_record(json.dumps(record)) + + @pytest.mark.parametrize( "row", [ @@ -178,6 +231,12 @@ def test_build_record_rejects_malformed_or_tampered(mutator: Callable[..., objec with pytest.raises(pfb_pkg.PkgError): pfb_pkg.validate_build_record(record) +def test_build_record_rejects_unknown_field_with_valid_digest() -> None: + record = _record(unknown=True) + + with pytest.raises(pfb_pkg.PkgError, match="exact fields"): + pfb_pkg.validate_build_record(record) + def test_build_record_nightly_timestamp_sha_and_null_rules() -> None: for value in ( diff --git a/tests/test_publish_release.py b/tests/test_publish_release.py index ee7eb38..c677881 100644 --- a/tests/test_publish_release.py +++ b/tests/test_publish_release.py @@ -1802,6 +1802,112 @@ def test_handoff_route_matrix_empty_array_rejected(self) -> None: ) self.assertIn("route_matrix must be a non-empty JSON array", str(ctx.exception)) + def _null_builder_handoff(self, rows: Sequence[dict[str, object]]) -> Path: + handoff = _write_handoff( + self.tmp / "pfblockerng-release-handoff.json", rows=rows, tag="v4.0.0.b1" + ) + payload = json.loads(handoff.read_text(encoding="utf-8")) + payload["dependency_builder"] = None + handoff.write_text(json.dumps(payload), encoding="utf-8") + return handoff + + def test_handoff_null_builder_without_extra_packages_accepted(self) -> None: + validated = trh.load_handoff( + self._null_builder_handoff((ROW_CE_NO_EXTRA,)), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + self.assertIsNone(validated["dependency_builder"]) + + def test_handoff_null_builder_with_extra_packages_rejected(self) -> None: + with self.assertRaises(trh.HandoffError) as ctx: + trh.load_handoff( + self._null_builder_handoff((ROW_CE,)), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + self.assertIn( + "dependency_builder is required when route_matrix contains extra packages", + str(ctx.exception), + ) + + def test_handoff_null_builder_with_mixed_extra_packages_rejected(self) -> None: + with self.assertRaises(trh.HandoffError) as ctx: + trh.load_handoff( + self._null_builder_handoff((ROW_CE_NO_EXTRA, ROW_PLUS_03_TWIN)), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + self.assertIn( + "dependency_builder is required when route_matrix contains extra packages", + str(ctx.exception), + ) + + def test_handoff_malformed_builder_without_extra_packages_rejected(self) -> None: + handoff = _write_handoff( + self.tmp / "pfblockerng-release-handoff.json", + rows=(ROW_CE_NO_EXTRA,), + tag="v4.0.0.b1", + ) + payload = json.loads(handoff.read_text(encoding="utf-8")) + payload["dependency_builder"] = {**_DEPENDENCY_BUILDER, "wheel": "not-a-version"} + handoff.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaises(trh.HandoffError) as ctx: + trh.load_handoff( + handoff, expected_release_tag="v4.0.0.b1", expected_source_sha="a" * 40 + ) + self.assertIn("dependency_builder", str(ctx.exception)) + + def test_handoff_valid_builder_without_extra_packages_accepted(self) -> None: + validated = trh.load_handoff( + _write_handoff( + self.tmp / "pfblockerng-release-handoff.json", + rows=(ROW_CE_NO_EXTRA,), + tag="v4.0.0.b1", + ), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + self.assertEqual(validated["dependency_builder"], _DEPENDENCY_BUILDER) + + def _builderless_record(self, row: dict[str, object]) -> dict[str, object]: + record = _record(row=row) + record.pop("dependency_builder") + record["build_input_digest"] = pfb_pkg.build_input_digest(record) + return record + + def test_null_builder_handoff_binds_builderless_empty_extras_records(self) -> None: + handoff = trh.load_handoff( + self._null_builder_handoff((ROW_CE_NO_EXTRA,)), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + trh.validate_build_records( + handoff, [self._builderless_record(ROW_CE_NO_EXTRA)] + ) + + def test_null_builder_handoff_rejects_builderless_nonempty_extras_record(self) -> None: + handoff = trh.load_handoff( + self._null_builder_handoff((ROW_CE_NO_EXTRA,)), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + with self.assertRaises(trh.BuildRecordIdentityError) as ctx: + trh.validate_build_records( + handoff, [self._builderless_record(ROW_CE)] + ) + self.assertEqual(ctx.exception.field, "dependency_builder") + + def test_null_builder_handoff_rejects_builder_record_mismatch(self) -> None: + handoff = trh.load_handoff( + self._null_builder_handoff((ROW_CE_NO_EXTRA,)), + expected_release_tag="v4.0.0.b1", + expected_source_sha="a" * 40, + ) + with self.assertRaises(trh.BuildRecordIdentityError) as ctx: + trh.validate_build_records(handoff, [_record(row=ROW_CE_NO_EXTRA)]) + self.assertEqual(ctx.exception.field, "dependency_builder") + def test_published_release_without_handoff_uses_pkg_compatibility_matrix( self, ) -> None: