From dc8b2e244744b226c29ca76ec38433013a748556 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 04:18:39 +0000 Subject: [PATCH 1/3] Build tagged releases and preserve publisher history --- .github/scripts/build_payload.py | 42 +++++++---- .github/scripts/resolve_release.py | 27 +++++++ .github/scripts/test_build_payload.py | 29 +++++++- .github/scripts/test_publish_contract.py | 94 ++++++++++++++++++++++++ .github/workflows/publish.yml | 47 ++++++------ README.md | 4 +- 6 files changed, 200 insertions(+), 43 deletions(-) create mode 100644 .github/scripts/resolve_release.py create mode 100644 .github/scripts/test_publish_contract.py diff --git a/.github/scripts/build_payload.py b/.github/scripts/build_payload.py index 5d98855..82c2064 100644 --- a/.github/scripts/build_payload.py +++ b/.github/scripts/build_payload.py @@ -25,12 +25,13 @@ import json import os from pathlib import Path -import re import subprocess import sys import urllib.error import urllib.request +from resolve_release import parse_release_tag + MAX_CONTENT_BYTES = 256 * 1024 @@ -67,10 +68,10 @@ def find_package_dir(ext_id: str) -> str: return matches[0] -def fetch_existing_releases(ext_id: str) -> list: +def fetch_existing_releases(ext_id: str) -> list | None: """Fetch the extension's currently published releases for merge history. - Returns [] for a new extension (API 404) or when no API token is + Returns None for a new extension (API 404) or when no API token is configured (offline use). Any other failure is fatal: silently submitting without history would wipe the extension's changelog. Entries are reduced to the release fields the API schema @@ -79,7 +80,7 @@ def fetch_existing_releases(ext_id: str) -> list: api_token = os.environ.get("API_TOKEN", "") api_base = os.environ.get("API_BASE_URL", "") if not api_token or not api_base: - return [] + return None http_request = urllib.request.Request( f"{api_base.rstrip('/')}/extensions/{ext_id}", @@ -90,7 +91,7 @@ def fetch_existing_releases(ext_id: str) -> list: body = json.load(response) except urllib.error.HTTPError as exc: if exc.code == 404: - return [] + return None fail(f"Could not fetch existing releases for '{ext_id}': HTTP {exc.code}") except Exception as exc: fail(f"Could not fetch existing releases for '{ext_id}': {exc}") @@ -130,7 +131,7 @@ def canonical_tag(tag: object, ext_id: str) -> object: return tag -def merge_releases(new_release: dict, ext_id: str) -> list: +def merge_releases(new_release: dict, ext_id: str, history: list) -> list: """Prepend the new release to the published history. Same-tag re-runs replace in place so a retried publish is idempotent; @@ -144,7 +145,7 @@ def merge_releases(new_release: dict, ext_id: str) -> list: merged = [new_release] merged.extend( entry - for entry in fetch_existing_releases(ext_id) + for entry in history if canonical_tag(entry.get("tag"), ext_id) != new_release["tag"] ) return merged[:MAX_RELEASES] @@ -152,8 +153,12 @@ def merge_releases(new_release: dict, ext_id: str) -> list: def main() -> None: ext_id, version, tag = sys.argv[1], sys.argv[2], sys.argv[3] - if not re.fullmatch(r"[a-z0-9_-]+", ext_id) or len(ext_id) > 64: - fail(f"Extension id {ext_id!r} is not a valid extension id (1–64 lowercase letters, digits, underscores, or hyphens)") + try: + target = parse_release_tag(tag) + except ValueError as exc: + fail(str(exc)) + if target != (ext_id, version): + fail("Extension id and version must match the release tag") ext_dir = find_package_dir(ext_id) manifest_path = os.path.join(ext_dir, "manifest.json") meta = json.loads(Path(manifest_path).read_text()) @@ -199,6 +204,10 @@ def main() -> None: if not readme.strip(): fail(f"{manifest_path} has an empty description and no README.md to fall back on") + # Zod measures JavaScript strings in UTF-16 code units, not code points. + if len(readme.encode("utf-16-le")) // 2 > 100_000: + fail(f"{readme_path} exceeds the API's 100,000-character README limit") + release = json.loads( subprocess.run( ["gh", "api", f"repos/{os.environ['GITHUB_REPOSITORY']}/releases/tags/{tag}"], @@ -239,6 +248,7 @@ def main() -> None: license_name = meta["license"] if not isinstance(license_name, str) or not license_name.strip(): fail(f"{manifest_path} license must be a non-empty license expression") + history = fetch_existing_releases(ext_id) content: dict = { "manifest_version": 1, "type": meta["type"], @@ -246,7 +256,7 @@ def main() -> None: "requires": requires, "name": meta["name"], "description": meta["description"], - "releases": merge_releases(new_release, ext_id), + "releases": merge_releases(new_release, ext_id, history or []), "license": license_name, "readme": readme, "source": {"type": "github", "repo": os.environ.get("GITHUB_REPOSITORY", "FOSSBilling/modules")}, @@ -263,14 +273,16 @@ def main() -> None: create_content = dict(content) create_content["id"] = ext_id - size = len(json.dumps(create_content).encode()) + size = len(json.dumps(create_content, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) if size > MAX_CONTENT_BYTES: fail("Built payload exceeds the 256 KiB content limit") - with open("payload-create.json", "w") as handle: - json.dump(create_content, handle) - with open("payload-update.json", "w") as handle: - json.dump(content, handle) + with open("payload-create.json", "w", encoding="utf-8") as handle: + json.dump(create_content, handle, ensure_ascii=False, separators=(",", ":")) + with open("payload-update.json", "w", encoding="utf-8") as handle: + json.dump(content, handle, ensure_ascii=False, separators=(",", ":")) + with open("payload-target.json", "w") as handle: + json.dump({"exists": history is not None}, handle) print(f"Built payloads for {ext_id} {version} ({size} bytes)") diff --git a/.github/scripts/resolve_release.py b/.github/scripts/resolve_release.py new file mode 100644 index 0000000..11bd889 --- /dev/null +++ b/.github/scripts/resolve_release.py @@ -0,0 +1,27 @@ +"""Resolve a release tag without interpolating user input into shell code.""" +import json +import os +from pathlib import Path +import re + + +SCHEMA = json.loads((Path(__file__).resolve().parents[2] / "manifest.schema.json").read_text()) + + +def parse_release_tag(tag: str) -> tuple[str, str]: + match = re.fullmatch(r"([a-z0-9_-]{1,64})-v(.+)", tag) + if not match or len(match[2]) > 100 or not re.fullmatch(SCHEMA["properties"]["version"]["pattern"], match[2]): + raise ValueError("Release tag must match -v") + return match[1], match[2] + + +def main() -> None: + tag = os.environ["RELEASE_TAG"] + ext_id, version = parse_release_tag(tag) + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"tag={tag}\nextension_id={ext_id}\nversion={version}\n") + print(f"Publishing {ext_id} version {version}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_build_payload.py b/.github/scripts/test_build_payload.py index 65cc1c3..edad533 100644 --- a/.github/scripts/test_build_payload.py +++ b/.github/scripts/test_build_payload.py @@ -41,7 +41,7 @@ def setUp(self): def build(self, history=None): (self.package / "manifest.json").write_text(json.dumps(self.meta)) - with patch.dict(os.environ, {"GITHUB_REPOSITORY": "FOSSBilling/modules"}), patch.object(publisher.sys, "argv", ["build_payload.py", "example_module", "1.2.3", "example_module-v1.2.3"]), patch.object(publisher.subprocess, "run", return_value=SimpleNamespace(stdout=json.dumps(self.release))), patch.object(publisher, "fetch_existing_releases", return_value=history or []): + with patch.dict(os.environ, {"GITHUB_REPOSITORY": "FOSSBilling/modules"}), patch.object(publisher.sys, "argv", ["build_payload.py", "example_module", "1.2.3", "example_module-v1.2.3"]), patch.object(publisher.subprocess, "run", return_value=SimpleNamespace(stdout=json.dumps(self.release))), patch.object(publisher, "fetch_existing_releases", return_value=history): publisher.main() return json.loads(Path("payload-create.json").read_text()), json.loads(Path("payload-update.json").read_text()) @@ -78,6 +78,22 @@ def test_requires_a_distributable_and_matching_version(self): with self.assertRaises(SystemExit): self.build() + def test_readme_limit_matches_api_utf16_length(self): + (self.package / "README.md").write_text("x" * 100_001) + with self.assertRaises(SystemExit): + self.build() + (self.package / "README.md").write_text("😀" * 50_001) + with self.assertRaises(SystemExit): + self.build() + (self.package / "README.md").write_text("😀" * 50_000) + self.build() + + def test_target_comes_from_the_history_read(self): + self.build() + self.assertEqual(json.loads(Path("payload-target.json").read_text()), {"exists": False}) + self.build([{"tag": "1.0.0", "requires": self.meta["requires"]}]) + self.assertEqual(json.loads(Path("payload-target.json").read_text()), {"exists": True}) + def test_requires_php(self): del self.meta["requires"]["php"] with self.assertRaises(SystemExit): @@ -87,6 +103,17 @@ def test_release_cap_is_preserved(self): history = [{"tag": f"0.0.{index}"} for index in range(101)] self.assertEqual(len(self.build(history)[0]["releases"]), 100) + def test_fetch_only_treats_404_as_a_new_extension(self): + for code in [404, 401, 429, 500]: + error = publisher.urllib.error.HTTPError("https://api.example.test", code, "error", {}, None) + self.addCleanup(error.close) + with self.subTest(code=code), patch.dict(os.environ, {"API_TOKEN": "test", "API_BASE_URL": "https://api.example.test"}), patch.object(publisher.urllib.request, "urlopen", side_effect=error): + if code == 404: + self.assertIsNone(publisher.fetch_existing_releases("example")) + else: + with self.assertRaises(SystemExit): + publisher.fetch_existing_releases("example") + def test_fetch_retains_requires_and_rejects_pre_migration_history(self): requirements = {"fossbilling": "^0.8 || ^1", "php": "^8", "modules": {"branding": "*"}} entry = {"tag": "1.0.0", "requires": requirements, "unknown": "ignored"} diff --git a/.github/scripts/test_publish_contract.py b/.github/scripts/test_publish_contract.py new file mode 100644 index 0000000..925f33b --- /dev/null +++ b/.github/scripts/test_publish_contract.py @@ -0,0 +1,94 @@ +"""Execute the workflow's submit shell offline against scripted API replies.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + +from resolve_release import parse_release_tag + + +ROOT = Path(__file__).resolve().parents[2] + + +def workflow_run(step): + source = (ROOT / ".github/workflows/publish.yml").read_text() + block = source.split(f" - name: {step}\n", 1)[1].split("\n - ", 1)[0] + return textwrap.dedent(block.split(" run: |\n", 1)[1]) + + +class ReleaseTargetTests(unittest.TestCase): + def test_valid_tags_preserve_prereleases_and_build_metadata(self): + for version in ["1.2.3", "1.2.3-v1.0.0", "1.2.3-alpha.1+build.4"]: + self.assertEqual(parse_release_tag(f"example_module-v{version}"), ("example_module", version)) + + def test_invalid_tags_are_rejected_before_outputs_or_checkout(self): + for tag in ["example-v01.2.3", "example-v1.2.3\nother=value", "example-v1.2.3;echo bad", "EXAMPLE-v1.2.3", "../example-v1.2.3"]: + with self.subTest(tag=tag), self.assertRaises(ValueError): + parse_release_tag(tag) + + +class PublishWorkflowTests(unittest.TestCase): + def run_submit(self, exists, replies, step="Submit revision (create or update)"): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "payload-target.json").write_text(json.dumps({"exists": exists})) + (root / "replies.json").write_text(json.dumps(replies)) + (root / "payload-create.json").write_text('{}') + (root / "payload-update.json").write_text('{}') + curl = root / "curl" + curl.write_text('''#!/usr/bin/env python3 +import json, pathlib, sys +args = sys.argv[1:] +root = pathlib.Path('.') +replies = json.loads((root / 'replies.json').read_text()) +response = replies.pop(0) +(root / 'replies.json').write_text(json.dumps(replies)) +method = args[args.index('-X') + 1] +with open('calls.txt', 'a') as calls: + calls.write(method + ' ' + args[-1] + '\\n') +assert method == response['method'], (method, response) +if '-o' in args: + (root / args[args.index('-o') + 1]).write_text(json.dumps(response['body'])) + (root / args[args.index('-D') + 1]).write_text('Retry-After: 0\\n') + print(response['code'], end='') +else: + print(json.dumps(response['body']) + '\\n' + str(response['code'])) +''') + curl.chmod(0o755) + env = {**os.environ, "PATH": f"{root}:{os.environ['PATH']}", "API_BASE_URL": "https://api.example.test/extensions/v2", "API_TOKEN": "offline", "EXT_ID": "example", "REVISION_ID": "revision", "GITHUB_OUTPUT": str(root / "outputs")} + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", workflow_run(step)], cwd=root, env=env, capture_output=True, text=True) + calls = (root / 'calls.txt').read_text().splitlines() + outputs = (root / 'outputs').read_text() if (root / 'outputs').exists() else '' + return result, calls, outputs + + def test_create_and_update_accept_the_merged_api_statuses(self): + for exists, method, code in [(False, "POST", 201), (True, "PUT", 202)]: + with self.subTest(method=method): + result, calls, outputs = self.run_submit(exists, [{"method": method, "code": code, "body": {"result": {"revision_id": "revision", "status": "pending"}}}]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(calls), 1) + self.assertEqual(outputs, 'revision_id=revision\n') + + def test_conflict_without_history_never_switches_to_update(self): + result, calls, _ = self.run_submit(False, [{"method": "POST", "code": 409, "body": {"error": {"message": "An extension with this id already exists"}}}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(calls), 1) + self.assertIn("refusing to overwrite releases", result.stdout) + + def test_explicit_machine_approval_matches_the_merged_api(self): + for code in [200, 403, 409]: + result, calls, _ = self.run_submit(True, [{"method": "POST", "code": code, "body": {"result": {"status": "approved"}}}], step="Approve revision") + self.assertEqual(result.returncode == 0, code == 200, result.stderr) + self.assertEqual(calls, ["POST https://api.example.test/extensions/v2/extensions/example/revisions/revision/approve?notify=false"]) + + def test_retries_preserve_update_method_and_payload(self): + result, calls, _ = self.run_submit(True, [{"method": "PUT", "code": 429, "body": {}}, {"method": "PUT", "code": 202, "body": {"result": {"revision_id": "revision", "status": "pending"}}}]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(calls), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d2ac9fa..3b20f4f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,23 +33,14 @@ jobs: - name: Resolve extension and version from tag id: target env: - GH_TOKEN: ${{ github.token }} - run: | - if [ "${{ github.event_name }}" = "release" ]; then - TAG="${{ github.event.release.tag_name }}" - else - TAG="${{ inputs.tag }}" - fi - if ! echo "$TAG" | grep -Eq '^.+-(v[0-9]+\.[0-9]+\.[0-9]+.*)$'; then - echo "::error::Tag '$TAG' does not match -v" - exit 1 - fi - EXT_ID="${TAG%-v*}" - VERSION="${TAG##*-v}" - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "extension_id=$EXT_ID" >> "$GITHUB_OUTPUT" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Publishing $EXT_ID version $VERSION" + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.tag }} + run: python3 .github/scripts/resolve_release.py + + - name: Checkout release packages + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ steps.target.outputs.tag }} + path: release-source - name: Fetch machine token id: token @@ -70,13 +61,18 @@ jobs: PUBLISHER_CLIENT_SECRET: ${{ secrets.MODULES_PUBLISHER_CLIENT_SECRET }} - name: Build extension payload + working-directory: release-source env: GH_TOKEN: ${{ github.token }} API_TOKEN: ${{ steps.token.outputs.access_token }} API_BASE_URL: ${{ env.API_BASE_URL }} - run: python3 .github/scripts/build_payload.py "${{ steps.target.outputs.extension_id }}" "${{ steps.target.outputs.version }}" "${{ steps.target.outputs.tag }}" + EXT_ID: ${{ steps.target.outputs.extension_id }} + VERSION: ${{ steps.target.outputs.version }} + RELEASE_TAG: ${{ steps.target.outputs.tag }} + run: python3 ../.github/scripts/build_payload.py "$EXT_ID" "$VERSION" "$RELEASE_TAG" - name: Submit revision (create or update) + working-directory: release-source id: submit env: API_TOKEN: ${{ steps.token.outputs.access_token }} @@ -100,8 +96,9 @@ jobs: && grep -i '^retry-after:' headers.txt | tail -1 | tr -d '\r' | awk '{print $2}' || echo 60 } - EXISTS=0 - api GET "/extensions/$EXT_ID" && [ "$HTTP_CODE" = "200" ] && EXISTS=1 + # Use the same read that supplied the payload's history. A second + # existence probe could switch an empty-history create into an update. + EXISTS=$(python3 -c "import json; print(int(json.load(open('payload-target.json'))['exists']))") attempt=0 while :; do @@ -119,17 +116,17 @@ jobs: attempt=$((attempt + 1)) continue fi - if [ "$HTTP_CODE" = "409" ] && [ "$EXISTS" = "0" ] && echo "$HTTP_BODY" | grep -q "already exists"; then - echo "Extension appeared between check and create, switching to update" - EXISTS=1 - continue + if [ "$HTTP_CODE" = "409" ] && [ "$EXISTS" = "0" ]; then + echo "::error::Create conflicted after a public 404: $HTTP_BODY" + echo "::error::History was unavailable. Resolve the hidden/reserved extension or retry after it becomes public; refusing to overwrite releases." + exit 1 fi break done if [ "$HTTP_CODE" = "409" ]; then echo "::error::API refused with 409: $HTTP_BODY" - echo "::error::A revision is likely already pending review — a moderator must approve or withdraw it before this release can publish." + echo "::error::A revision is likely already pending review — a moderator must approve or reject it before this release can publish." exit 1 fi if [ "$HTTP_CODE" != "$ok" ]; then diff --git a/README.md b/README.md index f9bca52..f8e4076 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Tag the release `-v` (e.g. `branding-v0.8.9`, version matching the package's `manifest.json`) and attach the distributable `.zip` as a release asset. The zip must contain the package directory contents at the archive root, with no wrapper folder, so it installs as-is. -Publishing a release runs `.github/workflows/publish.yml`, which sends the manifest v1 fields directly to the extensions API and approves the new revision. `type: "module"`, string licenses, optional `homepage_url`, package-relative `icon_url`, author metadata, and the complete `requires` object are preserved. A manual run is available via `workflow_dispatch` with the same tag. +Publishing a release runs `.github/workflows/publish.yml`, which sends the manifest v1 fields directly to the extensions API and approves the new revision. `type: "module"`, string licenses, optional `homepage_url`, package-relative `icon_url`, author metadata, and the complete `requires` object are preserved. A manual run is available via `workflow_dispatch` with the same tag. Both triggers read package files from that tag, while using publisher tooling from the reviewed workflow ref. FOSSBilling, PHP, and module constraints are retained verbatim, including compound Composer ranges. Each release stores its own requirements. Deploy the API manifest migration and matching extensions site before running this publisher; it refuses history without `requires` instead of discarding compatibility information. @@ -34,7 +34,7 @@ The release becomes a new extensions directory release entry keyed by the versio date, changelog link, and the distributable asset come from the GitHub release itself, and the entry is prepended to the extension's release history. A manual re-run via `workflow_dispatch` with the same tag -replaces the entry in place. +replaces the entry in place. A public 404 can also mean a delisted or unpublished extension: if creation then conflicts, publishing stops for manual correction instead of updating without its release history. Once the extension is public, retry to fetch its history first. ## Secrets From 8a893066462f24e656cf625113ba4461bcc0bc04 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 04:29:27 +0000 Subject: [PATCH 2/3] Isolate publisher Python imports from tagged content --- .github/scripts/test_publish_contract.py | 57 ++++++++++++++++++++++-- .github/workflows/publish.yml | 11 ++--- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/.github/scripts/test_publish_contract.py b/.github/scripts/test_publish_contract.py index 925f33b..1050688 100644 --- a/.github/scripts/test_publish_contract.py +++ b/.github/scripts/test_publish_contract.py @@ -1,6 +1,7 @@ """Execute the workflow's submit shell offline against scripted API replies.""" import json import os +import shutil from pathlib import Path import subprocess import tempfile @@ -16,7 +17,8 @@ def workflow_run(step): source = (ROOT / ".github/workflows/publish.yml").read_text() block = source.split(f" - name: {step}\n", 1)[1].split("\n - ", 1)[0] - return textwrap.dedent(block.split(" run: |\n", 1)[1]) + command = block.split(" run: ", 1)[1] + return textwrap.dedent(command[2:]) if command.startswith("|\n") else command.splitlines()[0] class ReleaseTargetTests(unittest.TestCase): @@ -31,16 +33,21 @@ def test_invalid_tags_are_rejected_before_outputs_or_checkout(self): class PublishWorkflowTests(unittest.TestCase): - def run_submit(self, exists, replies, step="Submit revision (create or update)"): + def run_submit(self, exists, replies, step="Submit revision (create or update)", hostile_module=None, after_api=False): with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / "payload-target.json").write_text(json.dumps({"exists": exists})) (root / "replies.json").write_text(json.dumps(replies)) (root / "payload-create.json").write_text('{}') (root / "payload-update.json").write_text('{}') + hostile_source = "from pathlib import Path\nPath('hostile-imported').write_text('executed')\nraise RuntimeError('tag-controlled import')\n" + if hostile_module and not after_api: + module = root / hostile_module + module.parent.mkdir(parents=True, exist_ok=True) + module.write_text(hostile_source) curl = root / "curl" - curl.write_text('''#!/usr/bin/env python3 -import json, pathlib, sys + curl.write_text('''#!/usr/bin/env -S python3 -I +import json, os, pathlib, sys args = sys.argv[1:] root = pathlib.Path('.') replies = json.loads((root / 'replies.json').read_text()) @@ -50,6 +57,10 @@ def run_submit(self, exists, replies, step="Submit revision (create or update)") with open('calls.txt', 'a') as calls: calls.write(method + ' ' + args[-1] + '\\n') assert method == response['method'], (method, response) +if os.environ.get('HOSTILE_MODULE'): + module = root / os.environ['HOSTILE_MODULE'] + module.parent.mkdir(parents=True, exist_ok=True) + module.write_text(os.environ['HOSTILE_SOURCE']) if '-o' in args: (root / args[args.index('-o') + 1]).write_text(json.dumps(response['body'])) (root / args[args.index('-D') + 1]).write_text('Retry-After: 0\\n') @@ -59,11 +70,49 @@ def run_submit(self, exists, replies, step="Submit revision (create or update)") ''') curl.chmod(0o755) env = {**os.environ, "PATH": f"{root}:{os.environ['PATH']}", "API_BASE_URL": "https://api.example.test/extensions/v2", "API_TOKEN": "offline", "EXT_ID": "example", "REVISION_ID": "revision", "GITHUB_OUTPUT": str(root / "outputs")} + if hostile_module and after_api: + env.update({"HOSTILE_MODULE": hostile_module, "HOSTILE_SOURCE": hostile_source}) result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", workflow_run(step)], cwd=root, env=env, capture_output=True, text=True) + self.assertFalse((root / 'hostile-imported').exists(), 'Tagged content executed during JSON parsing') calls = (root / 'calls.txt').read_text().splitlines() outputs = (root / 'outputs').read_text() if (root / 'outputs').exists() else '' return result, calls, outputs + def test_tagged_modules_cannot_hijack_either_submit_parser(self): + for module in ["json.py", "json/__init__.py"]: + for after_api in [False, True]: + with self.subTest(module=module, after_api=after_api): + result, _, outputs = self.run_submit(False, [{"method": "POST", "code": 201, "body": {"result": {"revision_id": "revision", "status": "pending"}}}], hostile_module=module, after_api=after_api) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(outputs, 'revision_id=revision\n') + + def test_build_uses_trusted_sibling_imports_despite_pythonpath(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + scripts = root / '.github/scripts' + scripts.mkdir(parents=True) + for name in ['build_payload.py', 'resolve_release.py']: + shutil.copyfile(ROOT / '.github/scripts' / name, scripts / name) + shutil.copyfile(ROOT / 'manifest.schema.json', root / 'manifest.schema.json') + release = root / 'release-source' + release.mkdir() + package = release / 'Branding' + package.mkdir() + for name in ['manifest.json', 'README.md']: + shutil.copyfile(ROOT / 'Branding' / name, package / name) + version = json.loads((package / 'manifest.json').read_text())['version'] + (release / 'json.py').write_text("from pathlib import Path\nPath('hostile-imported').write_text('executed')\nraise RuntimeError('tag-controlled import')\n") + bin_dir = root / 'bin' + bin_dir.mkdir() + gh = bin_dir / 'gh' + gh.write_text('#!/usr/bin/env -S python3 -I\nprint(' + repr(json.dumps({"published_at": "2026-01-01", "assets": [{"name": "release.zip", "browser_download_url": "https://example.test/release.zip"}]})) + ')\n') + gh.chmod(0o755) + env = {**os.environ, 'PATH': f"{bin_dir}:{os.environ['PATH']}", 'PYTHONPATH': str(release), 'API_TOKEN': 'offline', 'API_BASE_URL': '', 'GITHUB_REPOSITORY': 'FOSSBilling/modules', 'EXT_ID': 'branding', 'VERSION': version, 'RELEASE_TAG': f'branding-v{version}'} + result = subprocess.run(['bash', '-e', '-c', workflow_run('Build extension payload')], cwd=release, env=env, capture_output=True, text=True) + self.assertFalse((release / 'hostile-imported').exists(), 'Tagged module imported via PYTHONPATH') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads((release / 'payload-create.json').read_text())['id'], 'branding') + def test_create_and_update_accept_the_merged_api_statuses(self): for exists, method, code in [(False, "POST", 201), (True, "PUT", 202)]: with self.subTest(method=method): diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3b20f4f..5182027 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -34,7 +34,7 @@ jobs: id: target env: RELEASE_TAG: ${{ github.event.release.tag_name || inputs.tag }} - run: python3 .github/scripts/resolve_release.py + run: python3 -E -s .github/scripts/resolve_release.py - name: Checkout release packages uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -50,7 +50,7 @@ jobs: -d grant_type=client_credentials \ -d "scope=extensions:push extensions:publish" \ -d "resource=$API_AUDIENCE") - ACCESS=$(echo "$RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('access_token',''))") + ACCESS=$(echo "$RESP" | python3 -I -c "import json,sys; print(json.load(sys.stdin).get('access_token',''))") if [ -z "$ACCESS" ]; then echo "::error::Token request failed: $RESP" exit 1 @@ -69,7 +69,7 @@ jobs: EXT_ID: ${{ steps.target.outputs.extension_id }} VERSION: ${{ steps.target.outputs.version }} RELEASE_TAG: ${{ steps.target.outputs.tag }} - run: python3 ../.github/scripts/build_payload.py "$EXT_ID" "$VERSION" "$RELEASE_TAG" + run: python3 -E -s ../.github/scripts/build_payload.py "$EXT_ID" "$VERSION" "$RELEASE_TAG" - name: Submit revision (create or update) working-directory: release-source @@ -96,9 +96,10 @@ jobs: && grep -i '^retry-after:' headers.txt | tail -1 | tr -d '\r' | awk '{print $2}' || echo 60 } + # Isolate Python imports: tagged files are data, never parser code. # Use the same read that supplied the payload's history. A second # existence probe could switch an empty-history create into an update. - EXISTS=$(python3 -c "import json; print(int(json.load(open('payload-target.json'))['exists']))") + EXISTS=$(python3 -I -c "import json; print(int(json.load(open('payload-target.json'))['exists']))") attempt=0 while :; do @@ -133,7 +134,7 @@ jobs: echo "::error::Submit failed ($HTTP_CODE): $HTTP_BODY" exit 1 fi - REVISION_ID=$(echo "$HTTP_BODY" | python3 -c "import json,sys; print(json.load(sys.stdin)['result']['revision_id'])") + REVISION_ID=$(echo "$HTTP_BODY" | python3 -I -c "import json,sys; print(json.load(sys.stdin)['result']['revision_id'])") echo "revision_id=$REVISION_ID" >> "$GITHUB_OUTPUT" echo "Submitted revision $REVISION_ID (HTTP $HTTP_CODE)" From 5deaf01bf61faa8c67a5dfe35a87c4b32ff1d730 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 04:35:20 +0000 Subject: [PATCH 3/3] Qualify release checkout refs as tags --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5182027..02bbbbc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,7 +39,7 @@ jobs: - name: Checkout release packages uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - ref: ${{ steps.target.outputs.tag }} + ref: refs/tags/${{ steps.target.outputs.tag }} path: release-source - name: Fetch machine token