diff --git a/.github/scripts/build_payload.py b/.github/scripts/build_payload.py index 424eac8..5d98855 100644 --- a/.github/scripts/build_payload.py +++ b/.github/scripts/build_payload.py @@ -5,27 +5,26 @@ release's .zip asset, and writes two API content files: * ``payload-create.json`` for POST /extensions (includes the ``id`` the - catalogue requires on create), + API requires on create), * ``payload-update.json`` for PUT /extensions/{id} (identical except the ``id``, which that endpoint rejects as an unknown field). The new release entry is keyed by the plain version (``"0.8.9"``), not the -git tag (``"branding-v0.8.9"``): the catalogue sorts and compares releases +git tag (``"branding-v0.8.9"``): the API sorts and compares releases by semver tag on every read, and the extensions site derives the editable version from it. The entry is prepended to the extension's already published releases (fetched from the API when ``API_TOKEN`` is set) so the -catalogue keeps its changelog history; without a token only the new entry +API keeps its changelog history; without a token only the new entry is submitted. -The catalogue API speaks its own type vocabulary, so the manifest type is -translated here ("module" -> "mod"); unknown types fail loudly rather than -being passed through. +The manifest metadata is sent unchanged using the Extensions v2 contract. Usage: build_payload.py """ import json import os +from pathlib import Path import re import subprocess import sys @@ -35,12 +34,11 @@ MAX_CONTENT_BYTES = 256 * 1024 -# Catalogue cap on releases per extension (ExtensionContentSchema); older +# Extensions API cap on releases per extension (ExtensionContentSchema); older # entries past this are dropped from the submitted history. MAX_RELEASES = 100 -# Manifest (manifest.json v1) type -> catalogue API type. -TYPE_MAP = {"module": "mod"} +MANIFEST_TYPES = {"module", "theme", "payment-gateway", "server-manager", "domain-registrar", "translation"} def fail(message: str) -> None: @@ -57,7 +55,7 @@ def find_package_dir(ext_id: str) -> str: if not os.path.isdir(entry) or not os.path.isfile(manifest): continue try: - meta = json.load(open(manifest)) + meta = json.loads(Path(manifest).read_text()) except ValueError: fail(f"{manifest} is not valid JSON") if isinstance(meta, dict) and meta.get("id") == ext_id: @@ -69,28 +67,13 @@ def find_package_dir(ext_id: str) -> str: return matches[0] -def min_core_version(constraint: object, manifest_path: str) -> str: - """Reduce a `requires.fossbilling` constraint to the plain minimum - version the catalogue release record expects. Only simple >= pins are - accepted; anything fancier must be resolved by hand before releasing.""" - if not isinstance(constraint, str): - fail(f"{manifest_path} requires.fossbilling must be a version constraint string") - match = re.fullmatch(r">=\s*(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)", constraint.strip()) - if not match: - fail( - f"{manifest_path} requires.fossbilling must be a simple '>=X.Y.Z' constraint " - f"to publish, got {constraint!r}" - ) - return match.group(1) - - def fetch_existing_releases(ext_id: str) -> list: """Fetch the extension's currently published releases for merge history. Returns [] for a new extension (API 404) or when no API token is configured (offline use). Any other failure is fatal: silently - submitting without history would wipe the catalogue's changelog. - Entries are reduced to the release fields the catalogue schema + submitting without history would wipe the extension's changelog. + Entries are reduced to the release fields the API schema accepts, so a newer API response shape can never fail validation. """ api_token = os.environ.get("API_TOKEN", "") @@ -114,11 +97,13 @@ def fetch_existing_releases(ext_id: str) -> list: result = body.get("result") if isinstance(body, dict) else None releases = result.get("releases") if isinstance(result, dict) else None if not isinstance(releases, list): - fail(f"Catalogue response for '{ext_id}' has no releases list") + fail(f"Extensions API response for '{ext_id}' has no releases list") known = [] for entry in releases: if not isinstance(entry, dict): - fail(f"Catalogue response for '{ext_id}' has a malformed release entry") + fail(f"Extensions API response for '{ext_id}' has a malformed release entry") + if not isinstance(entry.get("requires"), dict) or not all(isinstance(entry["requires"].get(key), str) and entry["requires"][key] for key in ("fossbilling", "php")): + fail(f"Extensions API response for '{ext_id}' has a release without manifest requirements; deploy the API migration before publishing") known.append( { key: entry[key] @@ -127,7 +112,7 @@ def fetch_existing_releases(ext_id: str) -> list: "date", "download_url", "changelog_url", - "min_fossbilling_version", + "requires", ) if key in entry } @@ -149,7 +134,7 @@ def merge_releases(new_release: dict, ext_id: str) -> list: """Prepend the new release to the published history. Same-tag re-runs replace in place so a retried publish is idempotent; - the catalogue caps the list at its 100-release maximum, oldest dropped. + the API caps the list at its 100-release maximum, oldest dropped. A pre-fix entry keyed by git tag (`-v`) counts as the same release as its version, so re-publishing that version replaces the twin instead of doubling it; other legacy entries are preserved as-is. @@ -167,11 +152,11 @@ def merge_releases(new_release: dict, ext_id: str) -> list: def main() -> None: ext_id, version, tag = sys.argv[1], sys.argv[2], sys.argv[3] - if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9]+)*", ext_id) or len(ext_id) > 200: - fail(f"Extension id {ext_id!r} is not a valid catalogue id (lowercase alphanumeric segments separated by hyphens)") + if not re.fullmatch(r"[a-z0-9_-]+", ext_id) or len(ext_id) > 64: + fail(f"Extension id {ext_id!r} is not a valid extension id (1–64 lowercase letters, digits, underscores, or hyphens)") ext_dir = find_package_dir(ext_id) manifest_path = os.path.join(ext_dir, "manifest.json") - meta = json.load(open(manifest_path)) + meta = json.loads(Path(manifest_path).read_text()) for field in ( "manifest_version", @@ -182,7 +167,6 @@ def main() -> None: "description", "author", "license", - "homepage_url", "requires", ): if field not in meta: @@ -196,17 +180,18 @@ def main() -> None: if not isinstance(meta["requires"].get("fossbilling"), str): fail(f"{manifest_path} requires.fossbilling must be a version constraint string") - if meta["type"] not in TYPE_MAP: - fail(f"{manifest_path} has unsupported type {meta['type']!r} for catalogue submission") + if meta["type"] not in MANIFEST_TYPES: + fail(f"{manifest_path} has unsupported type {meta['type']!r} for extension submission") if meta["version"] != version: fail(f"{manifest_path} version {meta['version']!r} does not match release tag version {version!r}") requires = meta["requires"] - min_version = min_core_version(requires["fossbilling"], manifest_path) + if not isinstance(requires.get("php"), str) or not requires["php"].strip(): + fail(f"{manifest_path} requires.php must be a non-empty version constraint string") readme_path = os.path.join(ext_dir, "README.md") if os.path.isfile(readme_path): - readme = open(readme_path).read() + readme = Path(readme_path).read_text() if not readme.strip(): fail(f"{readme_path} is empty") else: @@ -238,7 +223,7 @@ def main() -> None: ) download_url = zips[0] - # The catalogue keys releases by plain semver version, not by git tag: + # The API keys releases by plain semver version, not by git tag: # sorting, latest-release resolution, and the directory's edit form # all compare `tag` with semver. The git tag stays reachable through # the changelog and download URLs. @@ -246,30 +231,35 @@ def main() -> None: "tag": version, "date": published_at, "download_url": download_url, - "min_fossbilling_version": min_version, + "requires": requires, } if html_url: new_release["changelog_url"] = html_url license_name = meta["license"] if not isinstance(license_name, str) or not license_name.strip(): - fail(f"{manifest_path} license must be a non-empty SPDX identifier") + fail(f"{manifest_path} license must be a non-empty license expression") content: dict = { - "type": TYPE_MAP[meta["type"]], + "manifest_version": 1, + "type": meta["type"], + "author": meta["author"], + "requires": requires, "name": meta["name"], "description": meta["description"], "releases": merge_releases(new_release, ext_id), - "website": meta["homepage_url"], - "license": {"name": license_name, "spdx_id": license_name}, + "license": license_name, "readme": readme, "source": {"type": "github", "repo": os.environ.get("GITHUB_REPOSITORY", "FOSSBilling/modules")}, "version": version, "download_url": download_url, } icon_url = meta.get("icon_url", "") - if isinstance(icon_url, str) and re.match(r"https?://", icon_url): + if isinstance(icon_url, str) and icon_url: content["icon_url"] = icon_url + if "homepage_url" in meta: + content["homepage_url"] = meta["homepage_url"] + create_content = dict(content) create_content["id"] = ext_id @@ -284,4 +274,5 @@ def main() -> None: print(f"Built payloads for {ext_id} {version} ({size} bytes)") -main() +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_build_payload.py b/.github/scripts/test_build_payload.py new file mode 100644 index 0000000..65cc1c3 --- /dev/null +++ b/.github/scripts/test_build_payload.py @@ -0,0 +1,101 @@ +"""Offline regression tests for release publishing; no credentials or network.""" +import importlib.util +from contextlib import redirect_stdout +import io +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location("publisher", Path(__file__).with_name("build_payload.py")) +publisher = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(publisher) + + +class PublisherTests(unittest.TestCase): + def setUp(self): + # Expected failures must not emit GitHub Actions error annotations. + self.enterContext(redirect_stdout(io.StringIO())) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.previous = os.getcwd() + self.addCleanup(os.chdir, self.previous) + os.chdir(self.temp.name) + self.package = Path("Example_module") + self.package.mkdir() + self.meta = { + "manifest_version": 1, "id": "example_module", "type": "module", + "name": "Example", "description": "Example module", "version": "1.2.3", + "author": {"name": "Author", "url": "https://example.com"}, + "license": "MIT OR Apache-2.0", "icon_url": "assets/icon.svg", + "requires": {"fossbilling": "^0.8.9 || ^1.0", "php": ">=8.3 <9", "modules": {"branding": "^1.2"}}, + } + (self.package / "README.md").write_text("# Example\n") + self.release = { + "published_at": "2026-11-04T00:00:00Z", "html_url": "https://github.com/FOSSBilling/modules/releases/tag/example_module-v1.2.3", + "assets": [{"name": "example.zip", "browser_download_url": "https://github.com/FOSSBilling/modules/releases/download/example_module-v1.2.3/example.zip"}], + } + + def build(self, history=None): + (self.package / "manifest.json").write_text(json.dumps(self.meta)) + with patch.dict(os.environ, {"GITHUB_REPOSITORY": "FOSSBilling/modules"}), patch.object(publisher.sys, "argv", ["build_payload.py", "example_module", "1.2.3", "example_module-v1.2.3"]), patch.object(publisher.subprocess, "run", return_value=SimpleNamespace(stdout=json.dumps(self.release))), patch.object(publisher, "fetch_existing_releases", return_value=history or []): + publisher.main() + return json.loads(Path("payload-create.json").read_text()), json.loads(Path("payload-update.json").read_text()) + + def test_preserves_manifest_metadata_and_compound_requirements(self): + create, update = self.build() + for field in self.meta: + self.assertEqual(create[field], self.meta[field]) + self.assertNotIn("website", create) + self.assertNotIn("homepage_url", create) + self.assertNotIn("id", update) + self.assertEqual(create["releases"][0]["requires"], self.meta["requires"]) + self.assertNotIn("min_fossbilling_version", create["releases"][0]) + self.assertEqual(create["readme"], "# Example\n") + self.assertEqual(create, {**update, "id": "example_module"}) + + def test_optional_homepage_and_icon(self): + self.meta["homepage_url"] = "https://example.com" + del self.meta["icon_url"] + create, _ = self.build() + self.assertEqual(create["homepage_url"], "https://example.com") + self.assertNotIn("icon_url", create) + + def test_reruns_replace_the_same_version_and_preserve_other_requirements(self): + old = {"tag": "1.0.0", "date": "2026-01-01", "download_url": "https://example.com/old.zip", "requires": {"fossbilling": "~0.7", "php": "^8.2", "modules": {"old": "^1"}}} + create, _ = self.build([{"tag": "example_module-v1.2.3"}, old]) + self.assertEqual([entry["tag"] for entry in create["releases"]], ["1.2.3", "1.0.0"]) + self.assertEqual(create["releases"][1], old) + + def test_requires_a_distributable_and_matching_version(self): + self.release["assets"] = [] + with self.assertRaises(SystemExit): + self.build() + self.meta["version"] = "2.0.0" + with self.assertRaises(SystemExit): + self.build() + + def test_requires_php(self): + del self.meta["requires"]["php"] + with self.assertRaises(SystemExit): + self.build() + + def test_release_cap_is_preserved(self): + history = [{"tag": f"0.0.{index}"} for index in range(101)] + self.assertEqual(len(self.build(history)[0]["releases"]), 100) + + def test_fetch_retains_requires_and_rejects_pre_migration_history(self): + requirements = {"fossbilling": "^0.8 || ^1", "php": "^8", "modules": {"branding": "*"}} + entry = {"tag": "1.0.0", "requires": requirements, "unknown": "ignored"} + with patch.dict(os.environ, {"API_TOKEN": "test", "API_BASE_URL": "https://api.example.test"}), patch.object(publisher.urllib.request, "urlopen", return_value=io.StringIO(json.dumps({"result": {"releases": [entry]}}))): + self.assertEqual(publisher.fetch_existing_releases("example"), [{"tag": "1.0.0", "requires": requirements}]) + with patch.dict(os.environ, {"API_TOKEN": "test", "API_BASE_URL": "https://api.example.test"}), patch.object(publisher.urllib.request, "urlopen", return_value=io.StringIO(json.dumps({"result": {"releases": [{"tag": "old", "min_fossbilling_version": "0.6"}]}}))): + with self.assertRaises(SystemExit): + publisher.fetch_existing_releases("example") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8db23e9..ddeb728 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -23,5 +23,5 @@ jobs: - name: Validate Manifest run: php tools/validate.php - - name: Compile check release tooling - run: python3 -m py_compile .github/scripts/build_payload.py \ No newline at end of file + - name: Test release tooling + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' diff --git a/.gitignore b/.gitignore index 9849e09..fbe3684 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ vendor/ -.DS_Store \ No newline at end of file +.DS_Store +__pycache__/ diff --git a/README.md b/README.md index fef337f..f9bca52 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FOSSBilling Modules -First-party extensions, published to the catalogue at +First-party extensions, published to the extensions directory at [extensions.fossbilling.org](https://extensions.fossbilling.org) by CI. ## Layout @@ -11,7 +11,7 @@ layout (`Branding/` for id `branding`): ```text Branding/ manifest.json # module manifest, v1 spec (required) - README.md # becomes the catalogue readme (required) + README.md # becomes the extensions directory readme (required) Service.php, Api/, Controller/, templates/, icon.svg, ... ``` @@ -26,18 +26,11 @@ Tag the release `-v` (e.g. `branding-v0.8.9`, version matching the package's `manifest.json`) and attach the distributable `.zip` as a release asset. The zip must contain the package directory contents at the archive root, with no wrapper folder, so it installs as-is. -Publishing a release runs `.github/workflows/publish.yml`, which builds the -catalogue payload from `manifest.json` (translating `type: module` to the -catalogue's `mod`, and `requires.fossbilling: ">=X.Y.Z"` to -`min_fossbilling_version: "X.Y.Z"`), submits the new version to the -extensions API and approves it. A manual run is available via -`workflow_dispatch` with the same tag. +Publishing a release runs `.github/workflows/publish.yml`, which sends the manifest v1 fields directly to the extensions API and approves the new revision. `type: "module"`, string licenses, optional `homepage_url`, package-relative `icon_url`, author metadata, and the complete `requires` object are preserved. A manual run is available via `workflow_dispatch` with the same tag. -Only simple `>=X.Y.Z` fossbilling constraints can be published; anything -fancier fails the payload build with an explicit error. `icon_url` is only -submitted when it is an absolute `https://` URL. +FOSSBilling, PHP, and module constraints are retained verbatim, including compound Composer ranges. Each release stores its own requirements. Deploy the API manifest migration and matching extensions site before running this publisher; it refuses history without `requires` instead of discarding compatibility information. -The release becomes a new catalogue release entry keyed by the version: +The release becomes a new extensions directory release entry keyed by the version: date, changelog link, and the distributable asset come from the GitHub release itself, and the entry is prepended to the extension's release history. A manual re-run via `workflow_dispatch` with the same tag