diff --git a/.github/workflows/test-actions.yml b/.github/workflows/test-actions.yml index e525b0f..ef91d37 100644 --- a/.github/workflows/test-actions.yml +++ b/.github/workflows/test-actions.yml @@ -174,7 +174,7 @@ jobs: - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - - run: pip install "PyYAML==6.0.2" "pytest>=8,<9" + - run: pip install "PyYAML==6.0.2" "pytest>=8,<9" "packaging==25.0" "jsonschema==4.25.1" shell: bash - run: python -m pytest -q shell: bash diff --git a/actions/winget-submit/action.yml b/actions/winget-submit/action.yml index 23a248a..97d26b6 100644 --- a/actions/winget-submit/action.yml +++ b/actions/winget-submit/action.yml @@ -33,26 +33,16 @@ inputs: description: "GitHub token for release asset download and winget-pkgs existence lookup" required: true winget-token: - description: "GitHub token used by wingetcreate to submit to microsoft/winget-pkgs" + description: "GitHub token used to create a fork and submit pull requests to microsoft/winget-pkgs" required: true runs: using: composite steps: - - name: Assert Windows runner - shell: bash - env: - RUNNER_OS_NAME: ${{ runner.os }} - run: | - set -euo pipefail - if [ "$RUNNER_OS_NAME" != "Windows" ]; then - echo "::error::winget-submit requires a Windows runner because wingetcreate.exe is Windows-only" - exit 1 - fi - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - shell: bash - run: pip install "PyYAML==6.0.2" + run: pip install "PyYAML==6.0.2" "packaging==25.0" "jsonschema==4.25.1" - shell: bash env: ACTION_PATH: ${{ github.action_path }} diff --git a/actions/winget-submit/test_winget_submit.py b/actions/winget-submit/test_winget_submit.py index dcf3d0b..6bce715 100644 --- a/actions/winget-submit/test_winget_submit.py +++ b/actions/winget-submit/test_winget_submit.py @@ -200,64 +200,6 @@ def capture_args(args): assert received["arm64_marker"] == "windows-arm64-setup.exe" -def test_load_wingetcreate_asset_uses_github_release_digest(): - def request_json(url, token): - return { - "assets": [ - { - "name": "wingetcreate.exe", - "browser_download_url": "https://example.test/wingetcreate.exe", - "digest": "sha256:" + "a" * 64, - } - ] - } - - asset = winget_submit.load_wingetcreate_asset("github-token", request_json=request_json) - - assert asset.url == "https://example.test/wingetcreate.exe" - assert asset.sha256 == "a" * 64 - - -def test_download_file_rejects_sha256_mismatch(tmp_path, monkeypatch): - class Response(io.BytesIO): - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - monkeypatch.setattr(winget_submit, "urlopen", lambda request, timeout: Response(b"not the expected bytes")) - dest = tmp_path / "wingetcreate.exe" - - with pytest.raises(winget_submit.SubmitError, match="checksum"): - winget_submit.download_file("https://example.test/wingetcreate.exe", dest, 1, "a" * 64) - - assert not dest.exists() - - -def test_download_file_accepts_matching_sha256(tmp_path, monkeypatch): - payload = b"expected bytes" - - class Response(io.BytesIO): - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - monkeypatch.setattr(winget_submit, "urlopen", lambda request, timeout: Response(payload)) - dest = tmp_path / "wingetcreate.exe" - - winget_submit.download_file( - "https://example.test/wingetcreate.exe", - dest, - 1, - hashlib.sha256(payload).hexdigest(), - ) - - assert dest.read_bytes() == payload - - def test_render_bootstrap_manifests_updates_values_without_mutating_source(tmp_path): package_id = "OpenCLICollective.codereview-cli" source = tmp_path / "tool" @@ -438,164 +380,88 @@ def test_render_bootstrap_manifests_rejects_output_inside_source(tmp_path): ) -def test_run_submit_existing_package_uses_update_command_and_token_contexts(monkeypatch): - assets = _assets() - calls = {} - - def load_release_assets(repo, final_tag, github_token): - calls["load_release_assets"] = (repo, final_tag, github_token) - return "checksums", {"asset": "url"} - - def resolve_windows_assets(checksums_text, release_assets, **_markers): - calls["resolve_windows_assets"] = (checksums_text, release_assets) - return assets - - def package_exists(package_id, github_token): - calls["package_exists"] = (package_id, github_token) - return True - - def download_file(url, dest, timeout_seconds, expected_sha256): - calls["download_file"] = (url, pathlib.Path(dest).name, timeout_seconds, expected_sha256) - - def run(command, check, timeout): - calls["run"] = (command, check, timeout) - - monkeypatch.setattr(winget_submit, "load_release_assets", load_release_assets) - monkeypatch.setattr(winget_submit, "resolve_windows_assets", resolve_windows_assets) - monkeypatch.setattr(winget_submit, "package_exists", package_exists) - monkeypatch.setattr( - winget_submit, - "load_wingetcreate_asset", - lambda github_token: winget_submit.DownloadAsset("https://example.test/wingetcreate.exe", "a" * 64), - ) - monkeypatch.setattr(winget_submit, "download_file", download_file) - monkeypatch.setattr(winget_submit.subprocess, "run", run) - - rc = winget_submit.run_submit(_args(bootstrap="true")) - - assert rc == 0 - assert calls["package_exists"] == ("OpenCLICollective.codereview-cli", "github-token") - assert calls["load_release_assets"] == ("open-cli-collective/codereview-cli", "v1.2.3", "github-token") - assert calls["download_file"] == ( - "https://example.test/wingetcreate.exe", - "wingetcreate.exe", - winget_submit.WINGETCREATE_DOWNLOAD_TIMEOUT_SECONDS, - "a" * 64, - ) - command, check, timeout = calls["run"] - assert check is True - assert timeout == winget_submit.WINGETCREATE_COMMAND_TIMEOUT_SECONDS - assert command[1:5] == ["update", "OpenCLICollective.codereview-cli", "--version", "1.2.3"] - assert command[5:8] == ["--urls", assets.x64.url, assets.arm64.url] - assert command[-2:] == ["--token", "winget-token"] - - -def test_run_submit_missing_package_with_bootstrap_submits_rendered_directory(monkeypatch): +def test_run_submit_existing_package_copies_upstream_manifests(monkeypatch): assets = _assets() calls = {} - - monkeypatch.setattr( - winget_submit, - "load_release_assets", - lambda repo, final_tag, github_token: ("checksums", {"asset": "url"}), - ) - monkeypatch.setattr( - winget_submit, - "resolve_windows_assets", - lambda checksums, release_assets, **_markers: assets, - ) - monkeypatch.setattr(winget_submit, "package_exists", lambda package_id, github_token: False) - monkeypatch.setattr( - winget_submit, - "download_file", - lambda url, dest, timeout_seconds, expected_sha256: calls.setdefault("download_file", pathlib.Path(dest).name), - ) - monkeypatch.setattr( - winget_submit, - "load_wingetcreate_asset", - lambda github_token: winget_submit.DownloadAsset("https://example.test/wingetcreate.exe", "a" * 64), - ) - - def render_bootstrap_manifests(package_id, version, working_dir, output_dir, assets): - calls["render"] = (package_id, version, pathlib.Path(working_dir), pathlib.Path(output_dir).exists()) - return [pathlib.Path(output_dir) / f"{package_id}.yaml"] - - def run(command, check, timeout): - calls["run"] = (command, check, timeout, pathlib.Path(command[-1]).exists()) - - monkeypatch.setattr(winget_submit, "render_bootstrap_manifests", render_bootstrap_manifests) - monkeypatch.setattr(winget_submit.subprocess, "run", run) - - rc = winget_submit.run_submit(_args(bootstrap="true")) - - assert rc == 0 - assert calls["render"] == ( - "OpenCLICollective.codereview-cli", - "1.2.3", - pathlib.Path("."), - True, - ) - command, check, timeout, rendered_dir_exists_during_submit = calls["run"] - assert check is True - assert timeout == winget_submit.WINGETCREATE_COMMAND_TIMEOUT_SECONDS - assert rendered_dir_exists_during_submit is True - assert command[1] == "submit" - assert command[2:5] == ["--prtitle", "New package: OpenCLICollective.codereview-cli version 1.2.3", "--token"] - assert command[5] == "winget-token" - assert command[6] == "--no-open" - - -def test_run_submit_missing_package_without_bootstrap_fails_before_download(monkeypatch): - calls = {} - - monkeypatch.setattr( - winget_submit, - "load_release_assets", - lambda repo, final_tag, github_token: ("checksums", {"asset": "url"}), - ) - monkeypatch.setattr(winget_submit, "resolve_windows_assets", lambda checksums, release_assets: _assets()) - monkeypatch.setattr(winget_submit, "package_exists", lambda package_id, github_token: False) - monkeypatch.setattr( - winget_submit, - "download_file", - lambda url, dest, timeout_seconds, expected_sha256: calls.setdefault("download_file", True), - ) - monkeypatch.setattr(winget_submit.subprocess, "run", lambda command, check, timeout: calls.setdefault("run", True)) - + monkeypatch.setattr(winget_submit, "load_release_assets", lambda *a: ("checksums", {})) + monkeypatch.setattr(winget_submit, "resolve_windows_assets", lambda *a, **kw: assets) + monkeypatch.setattr(winget_submit, "package_exists", lambda *a: True) + monkeypatch.setattr(winget_submit, "render_update_manifests", lambda directory, package_id, version, assets, token: [directory / "a.yaml"]) + monkeypatch.setattr(winget_submit, "validate_manifests", lambda *a: None) + monkeypatch.setattr(winget_submit, "submit_manifest_pr", lambda package_id, version, files, token, **kw: calls.setdefault("submit", (package_id, version, files, token, kw))) + assert winget_submit.run_submit(_args(bootstrap="true")) == 0 + assert calls["submit"][3:] == ("winget-token", {"new_package": False}) + + +def test_run_submit_bootstrap_uses_new_github_path(monkeypatch): + monkeypatch.setattr(winget_submit, "load_release_assets", lambda *a: ("checksums", {})) + monkeypatch.setattr(winget_submit, "resolve_windows_assets", lambda *a, **kw: _assets()) + monkeypatch.setattr(winget_submit, "package_exists", lambda *a: False) + monkeypatch.setattr(winget_submit, "render_bootstrap_manifests", lambda **kw: []) + monkeypatch.setattr(winget_submit, "upgrade_generated_manifests", lambda *a: [pathlib.Path("one.yaml")]) + calls = [] + monkeypatch.setattr(winget_submit, "validate_manifests", lambda *a: None) + monkeypatch.setattr(winget_submit, "submit_manifest_pr", lambda *a, **kw: calls.append((a, kw))) + assert winget_submit.run_submit(_args(bootstrap="true")) == 0 + assert calls[0][1] == {"new_package": True} + assert calls[0][0][3] == "winget-token" + + +def test_upgrade_generated_manifests_updates_schema_and_checks_types(tmp_path): + package_id = "OpenCLICollective.example" + for suffix, kind in [("", "version"), (".installer", "installer"), (".locale.en-US", "defaultLocale")]: + _write_manifest(tmp_path / f"{package_id}{suffix}.yaml", {"PackageIdentifier": package_id, "PackageVersion": "1.2.3", "ManifestType": kind, "ManifestVersion": "1.12.0"}) + paths = winget_submit.upgrade_generated_manifests(tmp_path, package_id, "1.2.3") + assert len(paths) == 3 + for path in paths: + content = path.read_text() + assert "1.28.0.schema.json" in content + assert yaml.safe_load(content)["ManifestVersion"] == "1.28.0" + + +def test_submit_manifest_pr_builds_tree_on_upstream_head(monkeypatch, tmp_path): + manifest = tmp_path / "OpenCLICollective.example.yaml" + manifest.write_text("content") + calls = [] + def api(url, token, method="GET", payload=None): + calls.append((url, token, method, payload)) + if url.endswith("/user"): return {"login": "alice"} + if url.endswith("/repos/alice/winget-pkgs"): return {"parent": {"full_name": "microsoft/winget-pkgs"}} + if url.endswith("/repos/microsoft/winget-pkgs"): return {"default_branch": "master"} + if "/git/ref/heads/" in url: return {"object": {"sha": "base"}} + if url.endswith("/git/commits/base"): return {"tree": {"sha": "base-tree"}} + if url.endswith("/git/trees"): return {"sha": "new-tree"} + if url.endswith("/git/commits"): return {"sha": "new-commit"} + if url.endswith("/git/refs"): return {} + if url.endswith("/pulls"): return {"html_url": "https://github.com/microsoft/winget-pkgs/pull/1"} + raise AssertionError(url) + monkeypatch.setattr(winget_submit, "github_api", api) + result = winget_submit.submit_manifest_pr("OpenCLICollective.example", "1.2.3", [manifest], "token", new_package=True) + assert result.endswith("/1") + tree = next(c[3] for c in calls if c[0].endswith("/git/trees")) + assert tree["base_tree"] == "base-tree" + assert tree["tree"][0]["path"] == "manifests/o/OpenCLICollective/example/1.2.3/OpenCLICollective.example.yaml" + commit = next(c[3] for c in calls if c[0].endswith("/git/commits")) + assert commit["parents"] == ["base"] + pr = next(c[3] for c in calls if c[0].endswith("/pulls")) + assert pr["base"] == "master" and pr["head"].startswith("alice:") + + +def test_missing_package_without_bootstrap_fails_before_release_lookup(monkeypatch): + monkeypatch.setattr(winget_submit, "package_exists", lambda *a: False) + monkeypatch.setattr(winget_submit, "load_release_assets", lambda *a: pytest.fail("unexpected release lookup")) with pytest.raises(winget_submit.SubmitError, match="bootstrap is not true"): winget_submit.run_submit(_args(bootstrap="false")) - assert "download_file" not in calls - assert "run" not in calls - - -def test_run_submit_unverifiable_package_lookup_fails_closed(monkeypatch): - calls = {} - - monkeypatch.setattr( - winget_submit, - "load_release_assets", - lambda repo, final_tag, github_token: ("checksums", {"asset": "url"}), - ) - monkeypatch.setattr(winget_submit, "resolve_windows_assets", lambda checksums, release_assets: _assets()) - - def package_exists(package_id, github_token): - raise winget_submit.SubmitError("could not verify whether package exists") - - monkeypatch.setattr(winget_submit, "package_exists", package_exists) - monkeypatch.setattr( - winget_submit, - "download_file", - lambda url, dest, timeout_seconds, expected_sha256: calls.setdefault("download_file", True), - ) - monkeypatch.setattr(winget_submit.subprocess, "run", lambda command, check, timeout: calls.setdefault("run", True)) +def test_unverifiable_package_lookup_fails_closed(monkeypatch): + def fail(*args): + raise winget_submit.SubmitError("could not verify") + monkeypatch.setattr(winget_submit, "package_exists", fail) + monkeypatch.setattr(winget_submit, "load_release_assets", lambda *a: pytest.fail("unexpected release lookup")) with pytest.raises(winget_submit.SubmitError, match="could not verify"): winget_submit.run_submit(_args(bootstrap="true")) - assert "download_file" not in calls - assert "run" not in calls - def _write_manifest(path, data): path.write_text(yaml.safe_dump(data, sort_keys=False)) @@ -669,3 +535,93 @@ def _args(bootstrap): github_token="github-token", winget_token="winget-token", ) + + + +def test_render_update_preserves_new_fields_and_updates_assets(monkeypatch, tmp_path): + import base64, json + package = "OpenCLICollective.codereview-cli" + records = { + f"{package}.yaml": {"PackageIdentifier": package, "PackageVersion": "1.0.0", "ManifestType": "version", "ManifestVersion": "1.28.0", "UnknownNewField": {"keep": "me"}}, + f"{package}.locale.en-US.yaml": {"PackageIdentifier": package, "PackageVersion": "1.0.0", "ManifestType": "defaultLocale", "ManifestVersion": "1.28.0"}, + f"{package}.installer.yaml": {"PackageIdentifier": package, "PackageVersion": "1.0.0", "ManifestType": "installer", "ManifestVersion": "1.28.0", "Installers": [{"Architecture": "x64", "InstallerUrl": "old", "InstallerSha256": "old", "UnknownField": "keep"}, {"Architecture": "arm64", "InstallerUrl": "old", "InstallerSha256": "old"}]}, + } + listing = [{"name": name, "url": f"https://test/{name}"} for name in records] + def fetch(url, token): + if url.endswith("/1.0.0"): return listing + if url.startswith("https://test/"): + name = url.rsplit("/", 1)[-1] + return {"content": base64.b64encode(json.dumps(records[name]).encode()).decode()} + return [{"name": "1.0.0", "type": "dir"}] + monkeypatch.setattr(winget_submit, "github_request_json", fetch) + output = winget_submit.render_update_manifests(tmp_path, package, "1.2.3", _assets(), "token") + assert len(output) == 3 + version = yaml.safe_load((tmp_path / f"{package}.yaml").read_text()) + assert version["UnknownNewField"] == {"keep": "me"} + assert version["PackageVersion"] == "1.2.3" + installer = yaml.safe_load((tmp_path / f"{package}.installer.yaml").read_text()) + assert installer["Installers"][0]["UnknownField"] == "keep" + assert installer["Installers"][0]["InstallerUrl"] == _assets().x64.url + assert installer["Installers"][1]["InstallerSha256"] == _assets().arm64.sha256 + + +def test_submit_manifest_pr_refuses_non_fork_before_writing(monkeypatch): + calls = [] + def api(url, token, method="GET", payload=None): + calls.append(method) + if url.endswith("/user"): + return {"login": "alice"} + return {"parent": {"full_name": "other/repo"}} + monkeypatch.setattr(winget_submit, "github_api", api) + with pytest.raises(winget_submit.SubmitError, match="not a fork"): + winget_submit.submit_manifest_pr("OpenCLICollective.example", "1.2.3", [], "token", new_package=True) + assert calls == ["GET", "GET"] + + +def test_schema_validation_rejects_invalid_manifest(monkeypatch, tmp_path): + import io, json + path = tmp_path / "invalid.yaml" + _write_manifest(path, {"ManifestType": "version", "PackageIdentifier": "wrong"}) + class Response(io.BytesIO): + pass + monkeypatch.setattr(winget_submit, "urlopen", lambda *args, **kwargs: Response(json.dumps({"type": "object", "required": ["PackageVersion"]}).encode())) + with pytest.raises(winget_submit.SubmitError, match="schema validation failed"): + winget_submit.validate_manifests([path]) + + +def test_render_update_rejects_unsupported_installer_layout(monkeypatch, tmp_path): + import base64, json + package = "OpenCLICollective.codereview-cli" + installer = {"PackageIdentifier": package, "PackageVersion": "1.0.0", "ManifestType": "installer", "ManifestVersion": "1.12.0", "Installers": [{"Architecture": "x64"}]} + def fetch(url, token): + if url.endswith("/1.0.0"): + return [{"name": package + ".installer.yaml", "url": "https://test/installer"}] + if url == "https://test/installer": + return {"content": base64.b64encode(json.dumps(installer).encode()).decode()} + return [{"name": "1.0.0", "type": "dir"}] + monkeypatch.setattr(winget_submit, "github_request_json", fetch) + with pytest.raises(winget_submit.SubmitError, match="exactly one x64 and one arm64"): + winget_submit.render_update_manifests(tmp_path, package, "1.2.3", _assets(), "token") + + +def test_submit_manifest_pr_uses_explicit_upstream_only_for_test(monkeypatch, tmp_path): + path = tmp_path / "OpenCLICollective.example.yaml" + path.write_text("content") + calls = [] + def api(url, token, method="GET", payload=None): + calls.append((url, method, payload)) + if url.endswith("/user"): return {"login": "alice"} + if url.endswith("/repos/alice/winget-pkgs"): + if len([x for x in calls if x[0].endswith("/repos/alice/winget-pkgs")]) == 1: + return {"parent": {"full_name": "microsoft/winget-pkgs"}} + return {"default_branch": "master"} + if "/git/ref/heads/" in url: return {"object": {"sha": "base"}} + if url.endswith("/git/commits/base"): return {"tree": {"sha": "base-tree"}} + if url.endswith("/git/trees"): return {"sha": "tree"} + if url.endswith("/git/commits"): return {"sha": "commit"} + if url.endswith("/git/refs"): return {} + if url.endswith("/pulls"): return {"html_url": "https://github.com/alice/winget-pkgs/pull/1"} + raise AssertionError(url) + monkeypatch.setattr(winget_submit, "github_api", api) + winget_submit.submit_manifest_pr("OpenCLICollective.example", "1.2.3", [path], "token", new_package=True, upstream_repo="alice/winget-pkgs") + assert any(url == "https://api.github.com/repos/alice/winget-pkgs/pulls" and method == "POST" for url, method, payload in calls) diff --git a/actions/winget-submit/winget_submit.py b/actions/winget-submit/winget_submit.py index 69663be..13a6409 100644 --- a/actions/winget-submit/winget_submit.py +++ b/actions/winget-submit/winget_submit.py @@ -3,12 +3,15 @@ from __future__ import annotations import argparse -import hashlib +import base64 +import datetime import json import os -import subprocess import sys import tempfile +import re +import time +import uuid from dataclasses import dataclass from pathlib import Path from urllib.error import HTTPError, URLError @@ -16,15 +19,15 @@ from urllib.request import Request, urlopen import yaml +import jsonschema +from packaging.version import InvalidVersion, Version MODE_UPDATE = "update" MODE_FIRST_SUBMISSION = "first-submission" MODE_MISSING_BOOTSTRAP_DISABLED = "missing-with-bootstrap-disabled" -WINGETCREATE_RELEASE_URL = "https://api.github.com/repos/microsoft/winget-create/releases/latest" HTTP_TIMEOUT_SECONDS = 30 -WINGETCREATE_DOWNLOAD_TIMEOUT_SECONDS = 60 -WINGETCREATE_COMMAND_TIMEOUT_SECONDS = 300 +TARGET_MANIFEST_VERSION = "1.28.0" WINGET_MANIFEST_TYPES = {"version", "installer", "defaultLocale", "locale"} DEFAULT_X64_MARKER = "windows_amd64.zip" DEFAULT_ARM64_MARKER = "windows_arm64.zip" @@ -53,12 +56,6 @@ class WindowsAssets: arm64: WindowsAsset -@dataclass(frozen=True) -class DownloadAsset: - url: str - sha256: str - - def package_id_to_winget_path(package_id: str) -> str: parts = [part for part in package_id.split(".") if part] if len(parts) < 2: @@ -112,23 +109,6 @@ def load_release_assets(repo: str, final_tag: str, github_token: str) -> tuple[s return checksums_text, download_urls -def load_wingetcreate_asset(github_token: str, request_json=None) -> DownloadAsset: - request_json = request_json or github_request_json - release = request_json(WINGETCREATE_RELEASE_URL, github_token) - for asset in release.get("assets") or []: - if asset.get("name") != "wingetcreate.exe": - continue - url = asset.get("browser_download_url") - digest = asset.get("digest") or "" - prefix = "sha256:" - if not url: - raise SubmitError("wingetcreate.exe release asset has no download URL") - if not digest.startswith(prefix): - raise SubmitError("wingetcreate.exe release asset has no SHA-256 digest") - return DownloadAsset(url=url, sha256=digest[len(prefix):]) - raise SubmitError("wingetcreate.exe asset not found in microsoft/winget-create latest release") - - def resolve_windows_assets( checksums_text: str, release_assets: dict[str, str], @@ -213,33 +193,167 @@ def render_bootstrap_manifests( return [dest for dest, _ in rendered] -def build_update_command(wingetcreate: Path, package_id: str, version: str, assets: WindowsAssets, token: str) -> list[str]: - return [ - str(wingetcreate), - "update", - package_id, - "--version", - version, - "--urls", - assets.x64.url, - assets.arm64.url, - "--submit", - "--token", - token, - ] - - -def build_submit_command(wingetcreate: Path, package_id: str, version: str, rendered_dir: Path, token: str) -> list[str]: - return [ - str(wingetcreate), - "submit", - "--prtitle", - f"New package: {package_id} version {version}", - "--token", - token, - "--no-open", - str(rendered_dir), - ] +def upgrade_generated_manifests(directory: Path, package_id: str, version: str) -> list[Path]: + """Upgrade generated manifests without passing them through the old serializer again.""" + paths = sorted(directory.glob("*.yaml")) + types = set() + for path in paths: + data = _load_manifest(path, package_id) + if str(data["PackageVersion"]) != version: + raise SubmitError(f"{path}: unexpected package version") + types.add(data.get("ManifestType")) + try: + if Version(str(data.get("ManifestVersion", ""))) > Version(TARGET_MANIFEST_VERSION): + raise SubmitError(f"{path}: refusing to downgrade newer manifest schema") + except InvalidVersion as exc: + raise SubmitError(f"{path}: invalid manifest schema version") from exc + data["ManifestVersion"] = TARGET_MANIFEST_VERSION + path.write_text(_dump_manifest(data, path), encoding="utf-8") + if not {"version", "installer", "defaultLocale"}.issubset(types): + raise SubmitError(f"{directory}: incomplete generated manifests") + return paths + + +def render_update_manifests(directory: Path, package_id: str, version: str, assets: WindowsAssets, token: str) -> list[Path]: + """Copy latest published manifests and update only the release-specific fields.""" + package_path = package_id_to_winget_path(package_id) + base_url = f"https://api.github.com/repos/microsoft/winget-pkgs/contents/{quote(package_path, safe='/')}" + entries = github_request_json(base_url, token) + versions = [] + for item in entries: + if item.get("type") != "dir": + continue + try: + versions.append((Version(item["name"]), item["name"])) + except InvalidVersion as exc: + raise SubmitError(f"cannot safely compare existing package version {item['name']!r}") from exc + if not versions: + raise SubmitError("existing package has no version directories") + try: + if Version(version) <= max(versions)[0]: + raise SubmitError(f"{version} is not newer than latest published version {max(versions)[1]}") + except InvalidVersion as exc: + raise SubmitError(f"cannot compare new package version {version!r}") from exc + current = max(versions)[1] + listing = github_request_json(f"{base_url}/{quote(current, safe='')}", token) + directory.mkdir(parents=True, exist_ok=True) + output = [] + types = set() + for item in listing: + name = item.get("name", "") + if not name.endswith(".yaml") or not name.startswith(package_id + "."): + continue + response = github_request_json(item["url"], token) + data = yaml.safe_load(base64.b64decode(response["content"])) + if not isinstance(data, dict) or data.get("PackageIdentifier") != package_id: + raise SubmitError(f"{name}: invalid package identifier or manifest") + if str(data.get("PackageVersion")) != current: + raise SubmitError(f"{name}: unexpected package version") + kind = data.get("ManifestType") + if kind not in WINGET_MANIFEST_TYPES or kind in types and kind != "locale": + raise SubmitError(f"{name}: unexpected or duplicate manifest type") + types.add(kind) + try: + if Version(str(data.get("ManifestVersion", ""))) > Version(TARGET_MANIFEST_VERSION): + raise SubmitError(f"{name}: refusing to downgrade newer manifest schema") + except InvalidVersion as exc: + raise SubmitError(f"{name}: invalid manifest schema version") from exc + data["PackageVersion"] = version + if kind in {"defaultLocale", "locale"}: + data.pop("ReleaseNotes", None) + data.pop("ReleaseNotesUrl", None) + if kind == "installer": + data.pop("ReleaseDate", None) + if "InstallerUrl" in data or "InstallerSha256" in data: + raise SubmitError("root-level installer URL or hash requires manual update") + installers = data.get("Installers") + if not isinstance(installers, list) or len(installers) != 2 or sorted(i.get("Architecture", "") for i in installers if isinstance(i, dict)) != ["arm64", "x64"]: + raise SubmitError("expected exactly one x64 and one arm64 installer; refusing automatic update") + for installer in installers: + installer.pop("ReleaseDate", None) + _update_installer_manifest(data, assets) + data["ManifestVersion"] = TARGET_MANIFEST_VERSION + dest = directory / name + dest.write_text(_dump_manifest(data, dest), encoding="utf-8") + output.append(dest) + if not {"version", "installer", "defaultLocale"}.issubset(types): + raise SubmitError("existing package does not have a complete set of multifile manifests") + return output + + +def validate_manifests(files: list[Path]) -> None: + schemas = {} + for path in files: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + kind = data["ManifestType"] + if kind not in schemas: + url = f"https://aka.ms/winget-manifest.{kind}.{TARGET_MANIFEST_VERSION}.schema.json" + with urlopen(Request(url, headers={"User-Agent": "open-cli-collective-winget-submit"}), timeout=HTTP_TIMEOUT_SECONDS) as response: + schemas[kind] = json.load(response) + # PyYAML recognizes dates; the schema expects their textual YAML representation. + instance = json.loads(json.dumps(data, default=lambda value: value.isoformat() if isinstance(value, (datetime.date, datetime.datetime)) else str(value))) + try: + jsonschema.Draft7Validator(schemas[kind]).validate(instance) + except jsonschema.ValidationError as exc: + raise SubmitError(f"{path.name}: schema validation failed at {'.'.join(map(str, exc.absolute_path))}: {exc.message}") from exc + + +def github_api(url: str, token: str, method: str = "GET", payload=None): + request = _github_request(url, token, "application/vnd.github+json") + request.method = method + if payload is not None: + request.data = json.dumps(payload).encode("utf-8") + request.add_header("Content-Type", "application/json") + try: + with urlopen(request, timeout=HTTP_TIMEOUT_SECONDS) as response: + return json.loads(response.read().decode("utf-8")) + except HTTPError as exc: + raise GitHubAPIError(exc.code, exc.read().decode("utf-8", "replace")) from exc + except (URLError, OSError) as exc: + raise GitHubAPIError(None, str(exc)) from exc + + +def submit_manifest_pr(package_id: str, version: str, files: list[Path], token: str, *, new_package: bool, upstream_repo: str = "microsoft/winget-pkgs", expected_fork_parent: str = "microsoft/winget-pkgs"): + """Create an upstream PR from a fresh tree based on the current upstream commit.""" + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]*", package_id) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.+-]*", version): + raise SubmitError("invalid package ID or version") + api = "https://api.github.com" + user = github_api(f"{api}/user", token)["login"] + fork_url = f"{api}/repos/{quote(user, safe='')}/winget-pkgs" + try: + fork = github_api(fork_url, token) + except GitHubAPIError as exc: + if exc.status != 404: + raise + github_api(f"{api}/repos/{upstream_repo}/forks", token, "POST", {"default_branch_only": True}) + fork = None + for _ in range(20): + time.sleep(3) + try: + fork = github_api(fork_url, token) + break + except GitHubAPIError as error: + if error.status != 404: + raise + if fork is None: + raise SubmitError("winget-pkgs fork was not ready after 60 seconds") + if (fork.get("parent") or {}).get("full_name", "").lower() != expected_fork_parent.lower(): + raise SubmitError(f"{user}/winget-pkgs is not a fork of {expected_fork_parent}") + upstream = github_api(f"{api}/repos/{upstream_repo}", token) + base = upstream["default_branch"] + head = github_api(f"{api}/repos/{upstream_repo}/git/ref/heads/{quote(base, safe='')}", token)["object"]["sha"] + commit = github_api(f"{api}/repos/{upstream_repo}/git/commits/{head}", token) + manifest_path = f"{package_id_to_winget_path(package_id)}/{version}" + tree = [{"path": f"{manifest_path}/{f.name}", "mode": "100644", "type": "blob", "content": f.read_text(encoding="utf-8")} for f in files] + tree_sha = github_api(f"{fork_url}/git/trees", token, "POST", {"base_tree": commit["tree"]["sha"], "tree": tree})["sha"] + message = f"{package_id} version {version}" + commit_sha = github_api(f"{fork_url}/git/commits", token, "POST", {"message": message, "tree": tree_sha, "parents": [head]})["sha"] + branch = f"open-cli-collective-{uuid.uuid4().hex}" + github_api(f"{fork_url}/git/refs", token, "POST", {"ref": f"refs/heads/{branch}", "sha": commit_sha}) + title = f"New package: {message}" if new_package else message + pr = github_api(f"{api}/repos/{upstream_repo}/pulls", token, "POST", {"title": title, "head": f"{user}:{branch}", "base": base, "body": f"Automated WinGet manifest submission for {message}."}) + _write_summary(f"WinGet pull request: {pr['html_url']}") + return pr["html_url"] def github_request_json(url: str, token: str): @@ -289,43 +403,16 @@ def run_submit(args) -> int: x64_marker=getattr(args, "x64_marker", DEFAULT_X64_MARKER), arm64_marker=getattr(args, "arm64_marker", DEFAULT_ARM64_MARKER), ) - wingetcreate_asset = load_wingetcreate_asset(args.github_token) - wingetcreate = Path.cwd() / "wingetcreate.exe" - download_file( - wingetcreate_asset.url, - wingetcreate, - WINGETCREATE_DOWNLOAD_TIMEOUT_SECONDS, - wingetcreate_asset.sha256, - ) - - if mode == MODE_UPDATE: - subprocess.run( - build_update_command(wingetcreate, args.package_id, args.version, assets, args.winget_token), - check=True, - timeout=WINGETCREATE_COMMAND_TIMEOUT_SECONDS, - ) - return 0 - with tempfile.TemporaryDirectory(prefix="winget-submit-") as tmp: - rendered_dir = Path(tmp) - render_bootstrap_manifests( - package_id=args.package_id, - version=args.version, - working_dir=args.working_dir, - output_dir=rendered_dir, - assets=assets, - ) - subprocess.run( - build_submit_command( - wingetcreate, - args.package_id, - args.version, - rendered_dir, - args.winget_token, - ), - check=True, - timeout=WINGETCREATE_COMMAND_TIMEOUT_SECONDS, - ) + root = Path(tmp) + rendered_dir = root / "rendered" + if mode == MODE_UPDATE: + files = render_update_manifests(rendered_dir, args.package_id, args.version, assets, args.github_token) + else: + files = render_bootstrap_manifests(package_id=args.package_id, version=args.version, working_dir=args.working_dir, output_dir=rendered_dir, assets=assets) + files = upgrade_generated_manifests(rendered_dir, args.package_id, args.version) + validate_manifests(files) + submit_manifest_pr(args.package_id, args.version, files, args.winget_token, new_package=mode == MODE_FIRST_SUBMISSION) return 0 @@ -397,28 +484,6 @@ def _github_request(url: str, token: str, accept: str) -> Request: return Request(url, headers=headers) -def download_file(url: str, dest: Path, timeout_seconds: int, expected_sha256: str) -> None: - request = Request(url, headers={"User-Agent": "open-cli-collective-winget-submit"}) - try: - hasher = hashlib.sha256() - with urlopen(request, timeout=timeout_seconds) as response: - with dest.open("wb") as fh: - while True: - chunk = response.read(1024 * 1024) - if not chunk: - break - hasher.update(chunk) - fh.write(chunk) - except (HTTPError, URLError, TimeoutError, OSError) as exc: - raise SubmitError(f"failed to download {url}: {exc}") from exc - actual = hasher.hexdigest() - if actual.lower() != expected_sha256.lower(): - dest.unlink(missing_ok=True) - raise SubmitError( - f"downloaded {url} checksum {actual} did not match expected {expected_sha256}" - ) - - def _parse_bool(value: str) -> bool: return str(value).strip().lower() in {"1", "true", "yes"} @@ -463,12 +528,7 @@ def main(argv: list[str] | None = None) -> int: except GitHubAPIError as exc: print(f"::error::GitHub API request failed with status {exc.status}: {exc}", file=sys.stderr) return 1 - except subprocess.TimeoutExpired as exc: - print(f"::error::wingetcreate timed out after {exc.timeout} seconds", file=sys.stderr) - return 1 - except subprocess.CalledProcessError as exc: - print(f"::error::wingetcreate failed with exit code {exc.returncode}", file=sys.stderr) - return exc.returncode or 1 + return 2