From f25ac3ec563c20e4068ea4bed46ef417cee8ec9b Mon Sep 17 00:00:00 2001 From: Quanzheng Long Date: Thu, 17 Sep 2026 21:10:56 -0700 Subject: [PATCH] fix: publish manifests for partial Server releases --- .../workflows/release-changed-components.yml | 21 +++----- .github/workflows/release-tooling-ci.yml | 2 +- CONTRIBUTING.md | 17 +++++-- Makefile | 5 +- release/compatibility/0.10.0.json | 48 ++++++++++++++++++ script/release/changed_components.py | 16 ++++++ .../release/changed_components_integ_test.py | 29 +++++++++++ script/release/compatibility_manifest.py | 50 ++++++++++++------- .../compatibility_manifest_integ_test.py | 36 +++++++++++++ 9 files changed, 186 insertions(+), 38 deletions(-) create mode 100644 release/compatibility/0.10.0.json diff --git a/.github/workflows/release-changed-components.yml b/.github/workflows/release-changed-components.yml index 37811aa2e..1c35ca09a 100644 --- a/.github/workflows/release-changed-components.yml +++ b/.github/workflows/release-changed-components.yml @@ -235,19 +235,13 @@ jobs: always() && needs.plan.result == 'success' && needs.create-releases.result == 'success' && - needs.plan.outputs.go == 'true' && - needs.plan.outputs.java == 'true' && - needs.plan.outputs.python == 'true' && - needs.plan.outputs.rust == 'true' && - needs.plan.outputs.typescript == 'true' && needs.plan.outputs.server == 'true' && - needs.plan.outputs.cli == 'true' && - needs.publish-java.result == 'success' && - needs.publish-python.result == 'success' && - needs.publish-rust.result == 'success' && - needs.publish-typescript.result == 'success' && + (needs.plan.outputs.java != 'true' || needs.publish-java.result == 'success') && + (needs.plan.outputs.python != 'true' || needs.publish-python.result == 'success') && + (needs.plan.outputs.rust != 'true' || needs.publish-rust.result == 'success') && + (needs.plan.outputs.typescript != 'true' || needs.publish-typescript.result == 'success') && needs.publish-server.result == 'success' && - needs.publish-cli.result == 'success' + (needs.plan.outputs.cli != 'true' || needs.publish-cli.result == 'success') uses: ./.github/workflows/compatibility-manifest.yml with: version: ${{ needs.plan.outputs.version }} @@ -325,9 +319,6 @@ jobs: exit 1 fi done - if [[ "${GO_SELECTED}" == "true" && "${JAVA_SELECTED}" == "true" && - "${PYTHON_SELECTED}" == "true" && "${RUST_SELECTED}" == "true" && - "${TYPESCRIPT_SELECTED}" == "true" && "${SERVER_SELECTED}" == "true" && - "${CLI_SELECTED}" == "true" && "${COMPATIBILITY_RESULT}" != "success" ]]; then + if [[ "${SERVER_SELECTED}" == "true" && "${COMPATIBILITY_RESULT}" != "success" ]]; then exit 1 fi diff --git a/.github/workflows/release-tooling-ci.yml b/.github/workflows/release-tooling-ci.yml index 0dccde5d6..5e0ac3cad 100644 --- a/.github/workflows/release-tooling-ci.yml +++ b/.github/workflows/release-tooling-ci.yml @@ -29,6 +29,6 @@ jobs: with: fetch-depth: 0 - name: Test release tooling - run: python3 -m unittest discover -s script/release -p '*_test.py' + run: make release-tooling-test - name: Validate workflow policy run: make ci-runner-check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9112eda47..0d8a10bd6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -166,17 +166,26 @@ Each component has its own version and tag prefix. Create a GitHub Release for t For coordinated releases, run **Release changed components** from the **main** branch and enter one semantic version. The workflow applies that version to every selected component. It compares each component with its own latest reachable release tag, uses that tag as the start of the component's generated release notes, preflights every selected target tag, creates the GitHub Releases, and directly invokes each publisher. The run succeeds only after every selected registry, Docker, CLI asset, and Homebrew publication succeeds. Go SDK publication is complete when its module tag and GitHub Release exist. -A coordinated Server, CLI, and all-SDK release also publishes a compatibility +Every coordinated release that selects Server also publishes a compatibility manifest. Add `release/compatibility/.json` before dispatching the release. The declaration records the reviewed protocol intervals, open-Flow compatibility, persistence compatibility, and rollout order. Release tooling -verifies those intervals against the tagged source, verifies that every -component tag resolves to one commit, reads the published CLI checksums and +verifies those intervals against each component's tagged source, reads the published CLI checksums and Server image digest, and uploads `dex-compatibility-v.json` to the Server GitHub Release. +For a partial release, include `componentVersions` with all seven component keys: +`server`, `cli`, `sdkGo`, `sdkJava`, `sdkPython`, `sdkRust`, and `sdkTypeScript`. +Changed components use the requested version; unchanged components retain their +published versions. The planner checks this declaration before creating tags. +The manifest's `sourceCommit` identifies Server; each component's tag identifies +its own source. Every declared client protocol must overlap the Server interval. +Omitting `componentVersions` declares that every component uses the release version. +An SDK-only or CLI-only release does not rewrite an existing Server manifest. + Use **Publish compatibility manifest** to backfill or reverify an already -published coordinated release. The workflow replaces only that release asset; +published Server release, including a partial release. The workflow adds the asset +when absent and requires byte-identical contents when it already exists; it never recreates component tags or republishes packages. Downstream systems must verify the downloaded manifest digest before changing a Server or SDK version. diff --git a/Makefile b/Makefile index 3ee3896df..80c960792 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,10 @@ GENERATED_CODE_PATHS := \ sdk-python/dex/dexpb \ sdk-typescript/src/gen -.PHONY: help ci-runner-check copyright copyright-check generated-code generated-code-check githooks docs-prose-check docs-prose-fix +.PHONY: help ci-runner-check copyright copyright-check generated-code generated-code-check githooks docs-prose-check docs-prose-fix release-tooling-test + +release-tooling-test: ## Verify component selection and immutable release manifests + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s script/release -p '*_test.py' help: ## Show targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-22s %s\n", $$1, $$2}' diff --git a/release/compatibility/0.10.0.json b/release/compatibility/0.10.0.json new file mode 100644 index 000000000..5d6a10f96 --- /dev/null +++ b/release/compatibility/0.10.0.json @@ -0,0 +1,48 @@ +{ + "schemaVersion": 1, + "release": "0.10.0", + "rolloutOrder": "server-first", + "runningFlowsCompatibility": "compatible", + "persistenceCompatibility": "compatible", + "protocol": { + "server": { + "minimum": 1, + "maximum": 1 + }, + "clients": { + "cli": { + "minimum": 1, + "maximum": 1 + }, + "sdkGo": { + "minimum": 1, + "maximum": 1 + }, + "sdkJava": { + "minimum": 1, + "maximum": 1 + }, + "sdkPython": { + "minimum": 1, + "maximum": 1 + }, + "sdkRust": { + "minimum": 1, + "maximum": 1 + }, + "sdkTypeScript": { + "minimum": 1, + "maximum": 1 + } + } + }, + "componentVersions": { + "server": "0.10.0", + "cli": "0.10.0", + "sdkGo": "0.10.0", + "sdkJava": "0.9.0", + "sdkPython": "0.9.0", + "sdkRust": "0.9.0", + "sdkTypeScript": "0.9.0" + } +} diff --git a/script/release/changed_components.py b/script/release/changed_components.py index 48d7ccfad..f8683569c 100644 --- a/script/release/changed_components.py +++ b/script/release/changed_components.py @@ -16,6 +16,8 @@ from dataclasses import dataclass from pathlib import Path +from compatibility_manifest import ManifestError, component_versions, load_declaration + VERSION_PATTERN = re.compile( r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" @@ -177,6 +179,20 @@ def write_outputs(output_path: Path, version: str) -> int: print(f"- {collision}", file=sys.stderr) return 1 + if any(component.key == "server" and selected for component, _, selected, _ in selections): + try: + declaration = load_declaration(Path("release/compatibility") / f"{version}.json", version) + versions = component_versions(declaration) + names = {"go": "sdkGo", "rust": "sdkRust", "java": "sdkJava", "python": "sdkPython", "typescript": "sdkTypeScript"} + for component, baseline, selected, target in selections: + key = names.get(component.key, component.key) + declared_tag = f"{component.tag_prefix}{versions[key]}" + if declared_tag != (target if selected else baseline): + raise ManifestError(f"declared {key} release does not match the release plan") + except ManifestError as error: + print(f"Compatibility preflight failed: {error}", file=sys.stderr) + return 1 + output_path.write_text("\n".join(output_lines) + "\n", encoding="utf-8") return 0 diff --git a/script/release/changed_components_integ_test.py b/script/release/changed_components_integ_test.py index 5fd375dfc..f046f4b05 100644 --- a/script/release/changed_components_integ_test.py +++ b/script/release/changed_components_integ_test.py @@ -9,6 +9,7 @@ from __future__ import annotations import subprocess +import json import tempfile import unittest from pathlib import Path @@ -124,6 +125,7 @@ def test_shared_native_change_triggers_dependent_sdks(self) -> None: def test_server_change_triggers_server_and_cli(self) -> None: self.change("server/main.go") + self.write_declaration("server", "cli") result, values = self.plan() self.assertEqual(result.returncode, 0, result.stderr) self.assert_selected(values, "server", "cli") @@ -155,10 +157,37 @@ def test_mixed_changes_select_multiple_releases(self) -> None: self.change("sdk-go/client.go") self.change("sdk-typescript/package.json") self.change("protos/api.proto") + self.write_declaration("go", "typescript", "server", "cli") result, values = self.plan() self.assertEqual(result.returncode, 0, result.stderr) self.assert_selected(values, "go", "typescript", "server", "cli") + def write_declaration(self, *selected: str) -> None: + source = SCRIPT.parents[2] / "release/compatibility/0.9.0.json" + declaration = json.loads(source.read_text(encoding="utf-8")) + declaration["release"] = "1.2.3" + names = {"go": "sdkGo", "rust": "sdkRust", "java": "sdkJava", "python": "sdkPython", "typescript": "sdkTypeScript"} + declaration["componentVersions"] = { + names.get(key, key): "1.2.3" if key in selected else "0.1.0" + for key in COMPONENT_KEYS + } + path = self.repository / "release/compatibility/1.2.3.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(declaration), encoding="utf-8") + + def test_server_release_requires_declaration_before_creating_tags(self) -> None: + self.change("server/main.go") + result, _ = self.plan() + self.assertNotEqual(result.returncode, 0) + self.assertIn("cannot read compatibility declaration", result.stderr) + + def test_partial_release_rejects_declaring_unpublished_sdk_version(self) -> None: + self.change("server/main.go") + self.write_declaration("server", "cli", "go") + result, _ = self.plan() + self.assertNotEqual(result.returncode, 0) + self.assertIn("declared sdkGo release does not match", result.stderr) + def test_missing_baseline_selects_first_release(self) -> None: self.git("tag", "-d", "sdk-go/v0.1.0") result, values = self.plan() diff --git a/script/release/compatibility_manifest.py b/script/release/compatibility_manifest.py index 21f259f22..f7d1aded9 100644 --- a/script/release/compatibility_manifest.py +++ b/script/release/compatibility_manifest.py @@ -132,7 +132,7 @@ def load_declaration(path: Path, version: str) -> dict[str, Any]: "persistenceCompatibility", "protocol", } - if not isinstance(declaration, dict) or set(declaration) != expected_keys: + if not isinstance(declaration, dict) or set(declaration) - {"componentVersions"} != expected_keys: raise ManifestError("compatibility declaration has unexpected fields") if declaration["schemaVersion"] != 1 or declaration["release"] != version: raise ManifestError("compatibility declaration version does not match the requested release") @@ -147,9 +147,23 @@ def load_declaration(path: Path, version: str) -> dict[str, Any]: }: raise ManifestError("persistenceCompatibility is invalid") validate_protocol_declaration(declaration["protocol"]) + component_versions(declaration) return declaration +def component_versions(declaration: dict[str, Any]) -> dict[str, str]: + versions = declaration.get("componentVersions", { + component.key: declaration["release"] for component in COMPONENTS + }) + if not isinstance(versions, dict) or set(versions) != {component.key for component in COMPONENTS}: + raise ManifestError("componentVersions must name every component") + if any(not isinstance(version, str) or not VERSION_PATTERN.fullmatch(version) for version in versions.values()): + raise ManifestError("componentVersions contains an invalid version") + if versions["server"] != declaration["release"]: + raise ManifestError("Server component version must match the release") + return versions + + def validate_protocol_declaration(protocol: Any) -> None: if not isinstance(protocol, dict) or set(protocol) != {"server", "clients"}: raise ManifestError("protocol declaration has unexpected fields") @@ -173,8 +187,8 @@ def validate_protocol_declaration(protocol: Any) -> None: raise ManifestError(f"protocol interval is invalid: {name}") -def resolve_release_commit(version: str, verify_releases: bool) -> tuple[str, dict[str, str]]: - tags = {component.key: component.tag(version) for component in COMPONENTS} +def resolve_release_commits(versions: dict[str, str], verify_releases: bool) -> tuple[dict[str, str], dict[str, str]]: + tags = {component.key: component.tag(versions[component.key]) for component in COMPONENTS} commits: dict[str, str] = {} for key, tag in tags.items(): commit = git("rev-parse", f"refs/tags/{tag}^{{commit}}") @@ -194,11 +208,7 @@ def resolve_release_commit(version: str, verify_releases: bool) -> tuple[str, di ) if metadata.get("tagName") != tag or metadata.get("targetCommitish") != commit: raise ManifestError(f"GitHub release target does not match tag commit: {tag}") - unique_commits = set(commits.values()) - if len(unique_commits) != 1: - details = ", ".join(f"{key}={value}" for key, value in sorted(commits.items())) - raise ManifestError(f"component release tags do not share one commit: {details}") - return unique_commits.pop(), tags + return commits, tags def read_protocol_interval(commit: str, key: str) -> dict[str, int]: @@ -214,14 +224,17 @@ def read_protocol_interval(commit: str, key: str) -> dict[str, int]: } -def verify_protocols(commit: str, declared: dict[str, Any]) -> None: +def verify_protocols(commits: dict[str, str], declared: dict[str, Any]) -> None: intervals = {"server": declared["server"], **declared["clients"]} for key, expected in intervals.items(): - actual = read_protocol_interval(commit, key) + actual = read_protocol_interval(commits[key], key) if actual != expected: raise ManifestError( f"protocol declaration does not match {key} source: expected {expected}, got {actual}" ) + server = declared["server"] + if max(actual["minimum"], server["minimum"]) > min(actual["maximum"], server["maximum"]): + raise ManifestError(f"{key} protocol does not overlap Server protocol") def parse_cli_checksums(path: Path, version: str) -> dict[str, str]: @@ -297,14 +310,15 @@ def build_manifest( if not VERSION_PATTERN.fullmatch(version): raise ManifestError(f"invalid semantic version: {version}") declaration = load_declaration(declaration_path, version) - source_commit, tags = resolve_release_commit(version, verify_releases) - verify_protocols(source_commit, declaration["protocol"]) - checksums = parse_cli_checksums(cli_checksums_path, version) + versions = component_versions(declaration) + commits, tags = resolve_release_commits(versions, verify_releases) + verify_protocols(commits, declaration["protocol"]) + checksums = parse_cli_checksums(cli_checksums_path, versions["cli"]) if verify_releases: verify_cli_release_assets(tags["cli"], checksums) components: dict[str, Any] = {} for component in COMPONENTS: - value: dict[str, Any] = {"version": version, "tag": tags[component.key]} + value: dict[str, Any] = {"version": versions[component.key], "tag": tags[component.key]} if component.key == "server": value["image"] = ( "docker.io/superdurable/dex-server@" + require_digest(server_digest) @@ -313,8 +327,8 @@ def build_manifest( value["checksums"] = checksums components[component.key] = value return { - **declaration, - "sourceCommit": source_commit, + **{key: value for key, value in declaration.items() if key != "componentVersions"}, + "sourceCommit": commits["server"], "components": components, } @@ -340,11 +354,13 @@ def main() -> int: ) temporary_directory: tempfile.TemporaryDirectory[str] | None = None try: + declaration = load_declaration(declaration_path, arguments.version) + versions = component_versions(declaration) server_digest = arguments.server_image_digest or resolve_server_digest(arguments.version) checksums_path = arguments.cli_checksums if checksums_path is None: temporary_directory, checksums_path = download_cli_checksums( - f"cli-v{arguments.version}" + f"cli-v{versions['cli']}" ) manifest = build_manifest( arguments.version, diff --git a/script/release/compatibility_manifest_integ_test.py b/script/release/compatibility_manifest_integ_test.py index 2e054c5b5..5b5735467 100644 --- a/script/release/compatibility_manifest_integ_test.py +++ b/script/release/compatibility_manifest_integ_test.py @@ -63,6 +63,8 @@ def test_builds_manifest_from_released_component_tags(self) -> None: self.write_checksums(directory), False, ) + digest = MODULE.write_manifest(directory / "manifest.json", manifest) + self.assertEqual(digest, "dc09203a2d785008d4449e23f70bd3598107e82d5f7934d86f49f1c534310906") self.assertEqual(manifest["sourceCommit"], SOURCE_COMMIT) self.assertEqual( manifest["components"]["server"]["image"], @@ -88,6 +90,40 @@ def test_rejects_protocol_declaration_drift(self) -> None: False, ) + def test_partial_release_uses_each_published_component_tag(self) -> None: + with tempfile.TemporaryDirectory() as directory_name: + directory = Path(directory_name) + checksums = {name.replace("0.9.0", "0.10.0"): value for name, value in CHECKSUMS.items()} + manifest = MODULE.build_manifest( + "0.10.0", + REPOSITORY / "release/compatibility/0.10.0.json", + SERVER_DIGEST, + self.write_checksums(directory, checksums), + False, + ) + self.assertEqual(manifest["sourceCommit"], "90dbc4ef121d575e4f79505cde0889cf49583fc2") + self.assertEqual(manifest["components"]["sdkGo"]["version"], "0.10.0") + self.assertEqual(manifest["components"]["sdkJava"]["tag"], "sdk-java/v0.9.0") + self.assertEqual(manifest["components"]["cli"]["checksums"], checksums) + + def test_partial_release_checks_retained_sdk_protocol_source(self) -> None: + declaration = json.loads((REPOSITORY / "release/compatibility/0.10.0.json").read_text()) + declaration["protocol"]["clients"]["sdkJava"]["maximum"] = 2 + with tempfile.TemporaryDirectory() as directory_name: + directory = Path(directory_name) + declaration_path = directory / "declaration.json" + declaration_path.write_text(json.dumps(declaration), encoding="utf-8") + with self.assertRaisesRegex(MODULE.ManifestError, "does not match sdkJava source"): + MODULE.build_manifest("0.10.0", declaration_path, SERVER_DIGEST, self.write_checksums(directory), False) + + def test_workflow_requires_manifest_for_partial_server_release(self) -> None: + workflow = (REPOSITORY / ".github/workflows/release-changed-components.yml").read_text() + condition = workflow.split(" publish-compatibility:", 1)[1].split(" uses:", 1)[0] + self.assertIn("needs.plan.outputs.server == 'true'", condition) + self.assertNotIn("needs.plan.outputs.java == 'true'", condition) + self.assertIn("needs.plan.outputs.java != 'true' || needs.publish-java.result == 'success'", condition) + self.assertIn('if [[ "${SERVER_SELECTED}" == "true" && "${COMPATIBILITY_RESULT}" != "success" ]]', workflow) + def test_rejects_missing_cli_archive(self) -> None: checksums = dict(CHECKSUMS) checksums.pop("dexcli_v0.9.0_linux_arm64.tar.gz")