diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edd89bb..aa310e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,10 +23,17 @@ jobs: shell: bash run: | set -euo pipefail - command -v sudo - command -v apt-get - sudo apt-get update - sudo apt-get install -y ripgrep + if ! command -v rg >/dev/null; then + release="ripgrep-14.1.0-x86_64-unknown-linux-musl" + archive="$RUNNER_TEMP/$release.tar.gz" + curl --fail --silent --show-error --location --retry 2 --max-time 60 \ + "https://github.com/BurntSushi/ripgrep/releases/download/14.1.0/$release.tar.gz" \ + --output "$archive" + printf '%s %s\n' 'f84757b07f425fe5cf11d87df6644691c644a5cd2348a2c670894272999d3ba7' "$archive" | sha256sum --check --status + tar -xzf "$archive" -C "$RUNNER_TEMP" "$release/rg" + export PATH="$RUNNER_TEMP/$release:$PATH" + printf '%s\n' "$RUNNER_TEMP/$release" >> "$GITHUB_PATH" + fi command -v rg - name: Verify public configuration scanner contract offline diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index eb9fa79..b119e88 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -3,6 +3,11 @@ name: Deploy Cloud Run on: workflow_dispatch: inputs: + release_cached_currency: + description: "Publish only the native currency field in the cached account archive." + required: false + default: false + type: boolean expected_sha: description: "Full commit SHA to deploy; must match the dispatched and approved branch head" required: true @@ -57,7 +62,7 @@ concurrency: jobs: sync-account-facts-configuration: name: Apply account-facts configuration (zero traffic) - if: github.event_name == 'workflow_dispatch' && inputs.sync_account_facts_configuration == true + if: github.event_name == 'workflow_dispatch' && inputs.release_cached_currency != true && inputs.sync_account_facts_configuration == true runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -216,7 +221,7 @@ jobs: stage-cached-balance-diagnostic: name: Stage cached-balance diagnostic (no traffic) - if: github.event_name == 'workflow_dispatch' && inputs.stage_cached_balance_diagnostic == true && inputs.sync_account_facts_configuration != true + if: github.event_name == 'workflow_dispatch' && inputs.release_cached_currency != true && inputs.stage_cached_balance_diagnostic == true && inputs.sync_account_facts_configuration != true runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -495,7 +500,7 @@ jobs: deploy-cloud-run: name: Deploy Cloud Run # Production deployment and environment sync require an explicit operator dispatch. - if: github.event_name == 'workflow_dispatch' && inputs.stage_cached_balance_diagnostic != true && inputs.sync_account_facts_configuration != true + if: github.event_name == 'workflow_dispatch' && inputs.release_cached_currency != true && inputs.stage_cached_balance_diagnostic != true && inputs.sync_account_facts_configuration != true runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -1702,3 +1707,46 @@ jobs: --async \ --quiet done <<< "${old_digests}" + + + release-cached-currency: + name: Release native currency in the cached archive + if: >- + github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && + github.run_attempt == 1 && inputs.release_cached_currency == true && + vars.ENABLE_GITHUB_CLOUD_RUN_DEPLOY == 'true' && + inputs.expected_sha == github.sha && inputs.approved_ref == github.ref && + !inputs.allow_configuration_sync && inputs.allow_traffic_promotion == true && !inputs.allow_cleanup && + !inputs.stage_cached_balance_diagnostic && !inputs.sync_account_facts_configuration + environment: firstrade-step5 + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.expected_sha }} + persist-credentials: false + - name: Recheck the approved branch after deployment review + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + run: | + set -euo pipefail + actual="$(git ls-remote origin refs/heads/main | cut -f1)" + if [ "$actual" != "$EXPECTED_SHA" ]; then + echo "The approved branch changed before the release." >&2 + exit 1 + fi + - uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ env.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@v3 + - name: Build and verify the archive-only revision + env: + CLOUD_RUN_REGION: ${{ vars.CLOUD_RUN_REGION }} + CLOUD_RUN_SERVICE: ${{ secrets.CLOUD_RUN_SERVICE }} + REVIEWED_WORKFLOW_SHA: ${{ inputs.expected_sha }} + run: python3 scripts/release_cached_currency.py diff --git a/scripts/release_cached_currency.py b/scripts/release_cached_currency.py new file mode 100644 index 0000000..ce4fbf1 --- /dev/null +++ b/scripts/release_cached_currency.py @@ -0,0 +1,196 @@ +"""Deploy only the already-reviewed native currency field in the cached archive.""" +from __future__ import annotations + +import ast +import copy +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import time +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +API = "https://run.googleapis.com/v2/" +MODULE = "application/session_check_service.py" +ANCHOR = '"total_equity": balances.get("provider_equity"),' +CURRENCY = '"currency": balances.get("currency"),' +STAGE = "configuration" +REVISION_METADATA = { + "name", "uid", "generation", "createTime", "updateTime", "deleteTime", "expireTime", + "launchStage", "service", "reconciling", "conditions", "observedGeneration", "logUri", + "satisfiesPzs", "scalingStatus", "containerStatuses", "creator", "etag", +} +TEMPLATE_METADATA = {"revision", "client", "clientVersion"} + + +def require(condition, reason): + if not condition: + raise ValueError(reason) + + +def patch_archive(source, reviewed): + def archive(text): + matches = [item for item in ast.parse(text).body if isinstance(item, ast.FunctionDef) + and item.name == "archive_cached_account_facts"] + require(len(matches) == 1, "cached_archive_function_missing") + return ast.get_source_segment(text, matches[0]) + + original = archive(source) + require(CURRENCY in archive(reviewed) and CURRENCY not in original and original.count(ANCHOR) == 1, + "cached_archive_patch_not_applicable") + updated = original.replace(ANCHOR, ANCHOR + "\n " + CURRENCY, 1) + candidate = source.replace(original, updated, 1) + require(archive(candidate) == updated, "cached_archive_patch_invalid") + return candidate + + +def verify_configuration(template, revision): + wanted = {k: v for k, v in template.items() if k not in TEMPLATE_METADATA} + actual = {k: v for k, v in revision.items() if k not in REVISION_METADATA | TEMPLATE_METADATA} + expected, = wanted["containers"] + found, = actual["containers"] + if "name" not in expected: + actual["containers"] = [{k: v for k, v in found.items() if k != "name"}] + require(wanted == actual, "revision_configuration_mismatch") + + +def wait(request, operation): + deadline = time.monotonic() + 240 + while not operation.get("done") and time.monotonic() < deadline: + time.sleep(4) + operation = request(operation["name"]) + require(operation.get("done") and not operation.get("error"), "cloud_run_operation_failed") + + +def revision_path(path, revision): + if re.fullmatch(r"[a-z0-9-]+", revision): + return path + "/revisions/" + revision + require(revision.startswith(path + "/revisions/"), "revision_not_in_service") + return revision + + +def rollout(request, path, build, *, verify_head=lambda: None): + global STAGE + baseline = request(path) + require(baseline["ingress"] == "INGRESS_TRAFFIC_INTERNAL_ONLY" and not baseline.get("reconciling"), + "serving_state_not_stable") + route, = [x for x in baseline["traffic"] if x.get("percent", 0) > 0] + status, = [x for x in baseline["trafficStatuses"] if x.get("percent", 0) > 0] + require(route["type"] == "TRAFFIC_TARGET_ALLOCATION_TYPE_REVISION" + and route["percent"] == status["percent"] == 100 and not route.get("tag"), + "existing_traffic_not_pinned") + old_path = revision_path(path, status["revision"]) + require(old_path == revision_path(path, route["revision"]), "traffic_configuration_drift") + active = request(old_path) + template = copy.deepcopy(baseline["template"]) + verify_configuration(template, active) + container, = template["containers"] + env = {x["name"]: x.get("value") for x in container["env"]} + require(env.get("FIRSTRADE_PERSIST_ACCOUNT_SNAPSHOT") == "true", "cached_archive_disabled") + before_image = container["image"] + STAGE = "build_verified_archive_patch" + container["image"] = build(before_image) + require(before_image != container["image"], "candidate_image_unchanged") + template.pop("revision", None) + template["labels"]["qsl-observation-patch"] = "cached-currency" + body = {"name": path, "etag": baseline["etag"], "template": template} + require(request(path)["etag"] == baseline["etag"], "baseline_changed") + STAGE = "stage_zero_traffic" + request(path + "?updateMask=template&validateOnly=true", "PATCH", body) + require(request(path)["etag"] == baseline["etag"], "baseline_changed") + wait(request, request(path + "?updateMask=template", "PATCH", body)) + staged = request(path) + require(staged["traffic"] == baseline["traffic"] and staged["trafficStatuses"] == baseline["trafficStatuses"] + and staged["ingress"] == baseline["ingress"] and not staged.get("reconciling"), "stage_changed_serving_state") + new_path = revision_path(path, staged["latestCreatedRevision"]) + require(new_path != old_path, "staged_revision_missing") + candidate = request(new_path) + require(not candidate.get("reconciling") and any( + x.get("type") == "Ready" and x.get("state") == "CONDITION_SUCCEEDED" for x in candidate.get("conditions", []) + ), "candidate_not_ready") + verify_configuration(template, candidate) + verify_configuration(staged["template"], candidate) + STAGE = "promote_currency_only" + verify_head() + traffic = [{**row, "revision": new_path.rsplit("/", 1)[-1]} if row == route else row for row in baseline["traffic"]] + payload = {"name": path, "etag": staged["etag"], "traffic": traffic} + request(path + "?updateMask=traffic&validateOnly=true", "PATCH", payload) + require(request(path)["etag"] == staged["etag"], "staged_service_changed") + wait(request, request(path + "?updateMask=traffic", "PATCH", payload)) + final = request(path) + routed, = [x for x in final["trafficStatuses"] if x.get("percent", 0) > 0] + require(revision_path(path, routed["revision"]) == new_path and routed["percent"] == 100 + and final["template"] == staged["template"] and final["ingress"] == baseline["ingress"] + and [x for x in final["traffic"] if x.get("tag")] == [x for x in baseline["traffic"] if x.get("tag")], + "final_serving_readback_mismatch") + + +def command(*args, timeout=45): + return subprocess.check_output(args, text=True, stderr=subprocess.DEVNULL, timeout=timeout).strip() + + +def build_image(base): + require(re.fullmatch(r"[a-z0-9-]+-docker.pkg.dev/[a-z0-9_./-]+@sha256:[a-f0-9]{64}", base), + "serving_image_not_immutable") + ref = base.split("@")[0] + host = ref.split("/")[0] + reviewed_sha = os.environ["REVIEWED_WORKFLOW_SHA"] + require(re.fullmatch(r"[a-f0-9]{40}", reviewed_sha), "reviewed_source_invalid") + tag = ref + ":cached-native-currency-" + reviewed_sha[:12] + command("gcloud", "auth", "configure-docker", host, "--quiet") + command("docker", "pull", base, timeout=180) + original = command("docker", "run", "--rm", "--network=none", "--entrypoint=cat", base, "/app/" + MODULE) + "\n" + patched = patch_archive(original, Path(MODULE).read_text()) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "archive.py").write_text(patched) + (root / "Dockerfile").write_text(f"FROM {base}\nCOPY archive.py /app/{MODULE}\n") + command("docker", "build", "--network=none", "-t", tag, directory, timeout=180) + old, new = json.loads(command("docker", "image", "inspect", base, tag)) + old_config = {k: v for k, v in old["Config"].items() if k != "Image"} + new_config = {k: v for k, v in new["Config"].items() if k != "Image"} + require(old_config == new_config and old["Architecture"] == new["Architecture"] and old["Os"] == new["Os"] + and new["RootFS"]["Layers"][:-1] == old["RootFS"]["Layers"], "image_changed_outside_archive_layer") + installed = command("docker", "run", "--rm", "--network=none", "--entrypoint=cat", tag, "/app/" + MODULE) + require(installed == patched.rstrip(), "built_archive_source_mismatch") + command("docker", "push", tag, timeout=180) + digest = command("gcloud", "artifacts", "docker", "images", "describe", tag, "--format=value(image_summary.digest)") + require(re.fullmatch(r"sha256:[a-f0-9]{64}", digest), "published_digest_invalid") + return ref + "@" + digest + + +def main(): + project, region, service = (os.environ[k] for k in ("GCP_PROJECT_ID", "CLOUD_RUN_REGION", "CLOUD_RUN_SERVICE")) + require(all(re.fullmatch(r"[a-z0-9-]+", x) for x in (project, region, service)), "service_binding_invalid") + token = command("gcloud", "auth", "print-access-token") + + def request(path, method="GET", body=None): + data = json.dumps(body).encode() if body is not None else None + headers = {"Authorization": "Bearer " + token, "Content-Type": "application/json"} + with urlopen(Request(API + path, data=data, headers=headers, method=method), timeout=30) as response: + result = json.load(response) + require(not result.get("error"), "cloud_run_api_error") + return result + + def verify_head(): + head = command("git", "ls-remote", "origin", "refs/heads/main").split() + require(len(head) == 2 and head[0] == os.environ["REVIEWED_WORKFLOW_SHA"], "reviewed_branch_changed") + + rollout(request, f"projects/{project}/locations/{region}/services/{service}", build_image, verify_head=verify_head) + print('{"cached_currency_release_verified":true,"runtime_configuration_preserved":true}') + + +if __name__ == "__main__": + try: + main() + except Exception as error: + result = {"cached_currency_release_verified": False, "stage": STAGE, "error_type": type(error).__name__} + if isinstance(error, ValueError) and re.fullmatch(r"[a-z_]+", str(error)): + result["reason"] = str(error) + if isinstance(error, HTTPError): + result["http_status"] = error.code + print(json.dumps(result)) + raise SystemExit(1) from None diff --git a/tests/test_public_runtime_configuration_scan.py b/tests/test_public_runtime_configuration_scan.py index ecf944f..0ad34c6 100644 --- a/tests/test_public_runtime_configuration_scan.py +++ b/tests/test_public_runtime_configuration_scan.py @@ -284,20 +284,19 @@ def test_unwritable_output_target_fails(self): self.assertNotEqual(result.returncode, 0) def test_tool_install_failure_fails_without_certification(self): - sudo = self.tools / "sudo" - sudo.write_text(f"#!{self.bash}\nexec \"$@\"\n", encoding="utf-8") - sudo.chmod(0o755) - apt_get = self.tools / "apt-get" - apt_get.write_text(f"#!{self.bash}\nexit 1\n", encoding="utf-8") - apt_get.chmod(0o755) + curl = self.tools / "curl" + curl.write_text(f"#!{self.bash}\nprintf 'synthetic download failure\\n' >&2\nexit 1\n", encoding="utf-8") + curl.chmod(0o755) self.output.unlink(missing_ok=True) result = self.run_script( - self.install_script, environment={"GITHUB_OUTPUT": str(self.output)} + self.install_script, + environment={"GITHUB_OUTPUT": str(self.output), "RUNNER_TEMP": str(self.root)}, ) + self.assertIn("synthetic download failure", result.stderr) self.assertNotEqual(result.returncode, 0) self.assertEqual(self.completion_output(), "") print( - f"Expected tool-install failure: synthetic apt-get; exit={result.returncode}; " + f"Expected tool-install failure: synthetic download; exit={result.returncode}; " "no completion certification emitted.", flush=True, ) diff --git a/tests/test_release_cached_currency.py b/tests/test_release_cached_currency.py new file mode 100644 index 0000000..6888191 --- /dev/null +++ b/tests/test_release_cached_currency.py @@ -0,0 +1,62 @@ +from pathlib import Path + +import pytest + +from scripts.release_cached_currency import MODULE, patch_archive + + +def test_archive_patch_preserves_every_other_byte(): + reviewed = Path(MODULE).read_text() + original = reviewed.replace(' "currency": balances.get("currency"),\n', "", 1) + assert original != reviewed + assert patch_archive(original, reviewed) == reviewed + + +def test_archive_patch_rejects_already_fixed_and_unknown_code(): + reviewed = Path(MODULE).read_text() + with pytest.raises(ValueError, match="patch_not_applicable"): + patch_archive(reviewed, reviewed) + with pytest.raises(ValueError, match="cached_archive_function_missing"): + patch_archive("pass\n", reviewed) + + +def test_rollout_preserves_existing_tags_and_runtime_configuration(): + import copy + + from scripts.release_cached_currency import rollout + + path = "projects/example/locations/example/services/example" + base = "example.invalid/app@sha256:" + "1" * 64 + next_image = "example.invalid/app@sha256:" + "2" * 64 + template = {"labels": {"release": "base"}, "serviceAccount": "example", "containers": [{ + "image": base, "env": [{"name": "FIRSTRADE_PERSIST_ACCOUNT_SNAPSHOT", "value": "true"}, + {"name": "RUNTIME_TARGET_ENABLED", "value": "true"}]}]} + pin = {"type": "TRAFFIC_TARGET_ALLOCATION_TYPE_REVISION", "revision": "example-old", "percent": 100} + tag = {"type": "TRAFFIC_TARGET_ALLOCATION_TYPE_REVISION", "revision": "example-test", "tag": "diag"} + service = {"ingress": "INGRESS_TRAFFIC_INTERNAL_ONLY", "traffic": [pin, tag], "etag": "old", + "trafficStatuses": [{"revision": "example-old", "percent": 100}, {"revision": "example-test", "tag": "diag"}], + "template": copy.deepcopy(template)} + writes = [] + + def request(ref, method="GET", payload=None): + if method == "PATCH": + writes.append((ref, copy.deepcopy(payload))) + assert payload["etag"] == service["etag"] + if ref.endswith("updateMask=template"): + service.update(template=payload["template"], latestCreatedRevision="example-new", etag="staged") + if ref.endswith("updateMask=traffic"): + service.update(traffic=payload["traffic"], trafficStatuses=[ + {"revision": "example-new", "percent": 100}, {"revision": "example-test", "tag": "diag"}]) + return {"done": True} + if ref == path: + return copy.deepcopy(service) + if ref.endswith("/example-old"): + return copy.deepcopy(template) + assert ref.endswith("/example-new") + return {**copy.deepcopy(service["template"]), "conditions": [{"type": "Ready", "state": "CONDITION_SUCCEEDED"}]} + + rollout(request, path, lambda image: next_image if image == base else "") + assert len(writes) == 4 + assert service["traffic"][1] == tag + assert service["template"]["containers"][0]["env"] == template["containers"][0]["env"] + assert service["template"]["serviceAccount"] == template["serviceAccount"]