From 960fdcf4cddbe8361644f92866006c833cea6909 Mon Sep 17 00:00:00 2001 From: KennySimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:06:37 +0800 Subject: [PATCH 1/2] fix(catalog): repair packages when the catalog is unchanged --- scripts/sync_catalog.py | 14 +++-- tests/test_sync_package_repair.py | 97 +++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 tests/test_sync_package_repair.py diff --git a/scripts/sync_catalog.py b/scripts/sync_catalog.py index f1bf1d3..a89f69e 100755 --- a/scripts/sync_catalog.py +++ b/scripts/sync_catalog.py @@ -143,12 +143,18 @@ def sync(root: Path, source: str, dry_run: bool) -> int: validate_catalog(mirrored) existing = load_json(root / "catalog.json") - if existing and canonical(existing) == canonical(mirrored): + packages_dir = root / "packages" + wanted = plan_downloads(src) + if existing and canonical(existing) == canonical(mirrored) and all( + item["yanked"] or ( + (packages_dir / item["name"]).is_file() + and sha256_file(packages_dir / item["name"]).lower() == item["shasum"] + ) + for item in wanted + ): print("catalog unchanged (ignoring generatedAt/catalogId/updatedAt)") return 0 - packages_dir = root / "packages" - wanted = plan_downloads(src) staging = Path(tempfile.mkdtemp(prefix="pi-mirror-")) staged_packages = staging / "packages" staged_packages.mkdir() @@ -157,7 +163,7 @@ def sync(root: Path, source: str, dry_run: bool) -> int: for item in wanted: dest = staged_packages / item["name"] local = packages_dir / item["name"] - if local.exists() and sha256_file(local).lower() == item["shasum"]: + if local.is_file() and sha256_file(local).lower() == item["shasum"]: shutil.copy2(local, dest) print(f"keep {item['name']}") continue diff --git a/tests/test_sync_package_repair.py b/tests/test_sync_package_repair.py new file mode 100644 index 0000000..c348cbd --- /dev/null +++ b/tests/test_sync_package_repair.py @@ -0,0 +1,97 @@ +"""An unchanged catalog must still have every required package on disk.""" +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location( + "sync_package_repair", Path(__file__).resolve().parents[1] / "scripts" / "sync_catalog.py", +) +sync = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sync) + + +class PackageRepairTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.packages = self.root / "packages" + self.packages.mkdir() + self.blob = b"verified-package" + self.source = "https://center.example/catalog.json" + self.src = { + "providerId": "official", + "artifactBaseUrl": "https://center.example/artifacts/", + "plugins": [{"id": "demo.hello", "versions": [{ + "version": "1.0.0", "shasum": sync.sha256_bytes(self.blob), "url": "hello.piplug", + }]}], + } + self.package = self.packages / "demo.hello-1.0.0.piplug" + self.write_catalog() + + def write_catalog(self): + self.catalog_bytes = (json.dumps(sync.rewrite_for_mirror(self.src)) + "\n").encode() + (self.root / "catalog.json").write_bytes(self.catalog_bytes) + + def run_sync(self, dry_run=False, blob=None): + def fetch(url, timeout): + if url == self.source: + return json.dumps(self.src).encode() + if url == self.source + ".sig": + return b"signature" + self.assertEqual(url, "https://center.example/artifacts/hello.piplug") + return self.blob if blob is None else blob + with patch.object(sync, "fetch", side_effect=fetch) as mocked: + self.assertEqual(sync.sync(self.root, self.source, dry_run), 0) + return mocked + + def test_missing_and_corrupt_packages_are_repaired(self): + for before in (None, b"corrupt"): + with self.subTest(before=before): + if before is None: + self.package.unlink(missing_ok=True) + else: + self.package.write_bytes(before) + self.run_sync() + self.assertEqual(self.package.read_bytes(), self.blob) + + def test_healthy_package_avoids_download_and_replacement(self): + self.package.write_bytes(self.blob) + fetched = self.run_sync() + self.assertEqual(fetched.call_count, 1) + self.assertEqual((self.root / "catalog.json").read_bytes(), self.catalog_bytes) + + def test_yanked_package_is_optional(self): + self.src["plugins"][0]["versions"][0]["yanked"] = True + self.write_catalog() + fetched = self.run_sync() + self.assertEqual(fetched.call_count, 1) + self.assertFalse(self.package.exists()) + + def test_failed_hash_verification_preserves_catalog_and_packages(self): + self.package.write_bytes(b"corrupt-local") + with self.assertRaisesRegex(SystemExit, "sha256 mismatch"): + self.run_sync(blob=b"corrupt-remote") + self.assertEqual(self.package.read_bytes(), b"corrupt-local") + self.assertEqual((self.root / "catalog.json").read_bytes(), self.catalog_bytes) + self.assertFalse((self.root / "catalog.json.tmp").exists()) + self.assertFalse((self.root / "packages.tmp").exists()) + + def test_dry_run_checks_repair_without_writing(self): + for before in (None, b"corrupt"): + with self.subTest(before=before): + if before is None: + self.package.unlink(missing_ok=True) + else: + self.package.write_bytes(before) + fetched = self.run_sync(dry_run=True) + self.assertEqual(fetched.call_count, 2) + self.assertEqual(self.package.read_bytes() if self.package.exists() else None, before) + self.assertEqual((self.root / "catalog.json").read_bytes(), self.catalog_bytes) + + +if __name__ == "__main__": + unittest.main() From 6994f6e20573173066c841af084985ac61656c78 Mon Sep 17 00:00:00 2001 From: KennySimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:11:21 +0800 Subject: [PATCH 2/2] refactor(catalog): isolate unchanged-mirror package checks --- scripts/sync_catalog.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/sync_catalog.py b/scripts/sync_catalog.py index a89f69e..dde6089 100755 --- a/scripts/sync_catalog.py +++ b/scripts/sync_catalog.py @@ -125,6 +125,16 @@ def plan_downloads(src: dict[str, Any]) -> list[dict[str, Any]]: return items +def required_packages_match(src: dict[str, Any], packages_dir: Path) -> bool: + for item in plan_downloads(src): + if item["yanked"]: + continue + local = packages_dir / item["name"] + if not local.is_file() or sha256_file(local).lower() != item["shasum"]: + return False + return True + + def load_json(path: Path) -> dict[str, Any] | None: if not path.exists(): return None @@ -143,18 +153,13 @@ def sync(root: Path, source: str, dry_run: bool) -> int: validate_catalog(mirrored) existing = load_json(root / "catalog.json") + if existing and canonical(existing) == canonical(mirrored): + if required_packages_match(src, root / "packages"): + print("catalog unchanged (ignoring generatedAt/catalogId/updatedAt)") + return 0 + packages_dir = root / "packages" wanted = plan_downloads(src) - if existing and canonical(existing) == canonical(mirrored) and all( - item["yanked"] or ( - (packages_dir / item["name"]).is_file() - and sha256_file(packages_dir / item["name"]).lower() == item["shasum"] - ) - for item in wanted - ): - print("catalog unchanged (ignoring generatedAt/catalogId/updatedAt)") - return 0 - staging = Path(tempfile.mkdtemp(prefix="pi-mirror-")) staged_packages = staging / "packages" staged_packages.mkdir()