From 8b935ac00609686ac9f6f208fc393b69bd7d1370 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:24:44 +0800 Subject: [PATCH 1/3] ci: Add private recovery evidence bootstrap --- .../workflows/bootstrap-recovery-evidence.yml | 37 +++ docs/ibkr-runtime-rollout.md | 59 +++++ scripts/recovery_evidence_bootstrap.py | 203 ++++++++++++++++ tests/test_recovery_evidence_bootstrap.py | 218 ++++++++++++++++++ 4 files changed, 517 insertions(+) create mode 100644 .github/workflows/bootstrap-recovery-evidence.yml create mode 100644 scripts/recovery_evidence_bootstrap.py create mode 100644 tests/test_recovery_evidence_bootstrap.py diff --git a/.github/workflows/bootstrap-recovery-evidence.yml b/.github/workflows/bootstrap-recovery-evidence.yml new file mode 100644 index 00000000..223e7a46 --- /dev/null +++ b/.github/workflows/bootstrap-recovery-evidence.yml @@ -0,0 +1,37 @@ +name: Bootstrap private recovery evidence + +on: + workflow_dispatch: + +permissions: + contents: read + id-token: write + +concurrency: + group: ibkr-retained-warm-recovery + cancel-in-progress: false + +jobs: + bootstrap: + if: github.event_name == 'workflow_dispatch' && github.run_attempt == 1 && github.ref == 'refs/heads/main' + environment: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@v3 + - name: Create private storage and retain stable pre-warm evidence + env: + IBKR_RECOVERY_BOOTSTRAP_JSON: ${{ secrets.IBKR_RECOVERY_BOOTSTRAP_JSON }} + RECOVERY_PROTECTED_ENVIRONMENT: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }} + run: python -m scripts.recovery_evidence_bootstrap diff --git a/docs/ibkr-runtime-rollout.md b/docs/ibkr-runtime-rollout.md index e7454612..6a309322 100644 --- a/docs/ibkr-runtime-rollout.md +++ b/docs/ibkr-runtime-rollout.md @@ -685,3 +685,62 @@ window, replacement-100 and warm-100 compensation, with independent minimum/cost readbacks. Exclude Peer1/Peer2/Peer4, all other services, Secret/IAM/Scheduler, environment sync, risk/execution switches, orders, deletion and automatic retry. Engineering acceptance does not authorize invoking this entry. + +### Private recovery evidence bootstrap + +`bootstrap-recovery-evidence.yml` is a manual, main-only preparatory entry in the +same protected Environment and concurrency group as warm recovery. It never +invokes recovery, changes Cloud Run, creates IAM/Secrets, or starts a warm +revision. Merging its definition does not authorize dispatch. Native Environment +approval and exact owner authority for the operator revision, target and storage +scope remain required; review and validate that Environment's protection before +any later dispatch. Reruns are refused. + +An authorized operator supplies `IBKR_RECOVERY_BOOTSTRAP_JSON` as a protected +Environment secret, never public dispatch inputs or repository Variables. It contains +`authority`, `protected_environment`, `project`, `region`, `service`, +`warm_revision`, `source_commit`, `operator_commit`, `image_digest`, `request_id`, +`bucket_suffix`, `issued_at`, `expires_at`, and `restore_warm_on_failure`. +Identity and image fields retain the recovery-card bindings above. The operator +commit must equal the workflow checkout; `issued_at` and `expires_at` have explicit +timezones and a positive window of at most 60 minutes. The suffix is 8–32 lowercase +letters/digits, the request identity follows the recovery-card grammar, and +`restore_warm_on_failure` must be true. Keep these values private. Configuring +these bindings is a separately authorized operation, not part of publishing code. + +The existing Environment workload identity needs only `run.services.get`, +`run.revisions.get`, `storage.buckets.create`, `storage.buckets.get`, +`storage.buckets.getIamPolicy`, `storage.objects.create`, and +`storage.objects.get` for this entry. Scope storage permissions to the dedicated +bucket where the provider supports that restriction; bucket creation requires the +approved project scope. No role/policy is created or granted by the entry. Missing +permissions stop execution rather than trigger a grant or an alternate identity. + +The script creates one dedicated regional Standard bucket with uniform +bucket-level access, enforced public access prevention, a 7-day object deletion +lifecycle, versioning disabled and soft deletion disabled. It verifies the bucket +and absence of public IAM members. Object deletion is lifecycle-managed and +asynchronous; seven days is not a precise deletion timestamp. Bucket deletion is +outside this entry. No existing bucket is adopted or repaired: creation is the +distributed single-use claim, including after runner restart. A conflict, timeout +or uncertain write stops without retry, cleanup or a second suffix. Reconcile the +original request privately before considering any separately approved new one. + +Fresh v1 service, v1 warm revision and v2 service snapshots retain resource sizes, +scaling, billing annotations and identity/configuration privately. Stable readbacks +must match before a card is uploaded and before its receipt is retained. Only +`gs:///s3-recovery//pre-warm-snapshot.json`, +`recovery-card.json` and `receipt.json` are written, with create-only generation +preconditions and exact byte readbacks. There is no public artifact, locator or +checksum output. An authorized protected read of `receipt.json` supplies the card +locator/checksum for the separate recovery gate. Partial artifacts or a missing +receipt are not success and cannot authorize recovery. + +The receipt explicitly leaves regional-price and incremental-budget qualification +`NOT_PROVEN`. These snapshots are inputs to that assessment, not a price quote, +US$1 budget proof, health qualification or recovery approval. Later recovery must +verify the exact merged operator revision, valid owner time/cost bounds, private +receipt and card checksum, fresh target state and native Environment approval. +Expiry stops bootstrap writes; this entry has no compensation because it never +changes warm/traffic state. It cannot extend a prior approval or automatically +launch recovery after merge. diff --git a/scripts/recovery_evidence_bootstrap.py b/scripts/recovery_evidence_bootstrap.py new file mode 100644 index 00000000..8019dc26 --- /dev/null +++ b/scripts/recovery_evidence_bootstrap.py @@ -0,0 +1,203 @@ +"""Private storage/snapshot bootstrap. Never writes Cloud Run or invokes recovery.""" +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import json +import os +import re +import subprocess +import sys +from urllib.parse import quote +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from scripts.mask_sync_identifiers import mask_identifiers + + +class BootstrapError(RuntimeError): + """Fixed diagnostic; provider data stays private.""" + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise BootstrapError('bootstrap provider redirect refused') + + +urlopen = build_opener(NoRedirect()).open + + +def require(value): + if not value: + raise BootstrapError('bootstrap binding or readback failed') + + +def encode(value): + return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + + +LIFECYCLE = {'rule': [{'action': {'type': 'Delete'}, 'condition': {'age': 7}}]} + + +class API: + """Single attempts with captured credentials/diagnostics; no SDK retries.""" + def __init__(self, config): + self.config = config + self.bucket = 'ibkr-recovery-evidence-' + config['bucket_suffix'] + + def __call__(self, kind, *, body=None, name=None, data=None): + c = self.config + base = ('https://' + c['region'] + '-run.googleapis.com/apis/serving.knative.dev/v1/' + 'namespaces/' + c['project']) + bucket = 'https://storage.googleapis.com/storage/v1/b/' + self.bucket + method = 'GET' + if kind == 'service': url = base + '/services/' + c['service'] + elif kind == 'warm': url = base + '/revisions/' + c['warm_revision'] + elif kind == 'minimum': + url = ('https://run.googleapis.com/v2/projects/' + c['project'] + + '/locations/' + c['region'] + '/services/' + c['service']) + elif kind == 'create_bucket': + url = 'https://storage.googleapis.com/storage/v1/b?project=' + c['project'] + method, data = 'POST', encode(body) + elif kind == 'bucket': url = bucket + elif kind == 'iam': url = bucket + '/iam' + elif kind == 'upload': + url = ('https://storage.googleapis.com/upload/storage/v1/b/' + self.bucket + + '/o?uploadType=media&ifGenerationMatch=0&name=' + quote(name, safe='')) + method = 'POST' + elif kind == 'download': url = bucket + '/o/' + quote(name, safe='') + '?alt=media' + else: raise BootstrapError('unsupported bootstrap operation') + try: + token = subprocess.run(['gcloud', 'auth', 'print-access-token'], capture_output=True, + text=True, check=True, timeout=15).stdout.strip() + require(token) + request = Request(url, data=data, method=method, + headers={'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/json'}) + with urlopen(request, timeout=30) as response: + result = response.read() + return result if kind == 'download' else json.loads(result) + except Exception: + raise BootstrapError('bootstrap provider operation failed or outcome unknown') from None + + +class Bootstrap: + def __init__(self, config, *, operator, environment, api=None, + now=lambda: datetime.now(timezone.utc)): + self.config, self.now = config, now + c = config + for key in ('project', 'region', 'service', 'warm_revision'): + require(isinstance(c[key], str) and re.fullmatch(r'[a-z][a-z0-9-]{0,62}', c[key])) + require(re.fullmatch(r'[a-z0-9][a-z0-9-]{0,30}', c['request_id'])) + require(re.fullmatch(r'[a-z0-9]{8,32}', c['bucket_suffix'])) + require(c['warm_revision'].startswith(c['service'] + '-')) + require(c['operator_commit'] == operator and re.fullmatch(r'[a-f0-9]{40}', operator)) + require(re.fullmatch(r'[a-f0-9]{40}', c['source_commit'])) + require(re.fullmatch(r'sha256:[a-f0-9]{64}', c['image_digest'])) + require(environment and c['protected_environment'] == environment and c['authority']) + require(c['restore_warm_on_failure'] is True) + self.issued = datetime.fromisoformat(c['issued_at']) + self.expires = datetime.fromisoformat(c['expires_at']) + require(self.issued.tzinfo is not None and self.expires.tzinfo is not None) + require(0 < (self.expires - self.issued).total_seconds() <= 3600) + require(len(c['service'] + '-rec-' + c['request_id']) <= 63) + self.valid() + self.bucket = 'ibkr-recovery-evidence-' + c['bucket_suffix'] + self.prefix = 's3-recovery/' + c['request_id'] + '/' + self.api = api or API(c) + + def valid(self): + require(self.issued <= self.now() < self.expires) + + def snapshot(self): + c = self.config + service, warm, minimum = (self.api(kind) for kind in ('service', 'warm', 'minimum')) + require(service['metadata']['name'] == c['service']) + require(service['metadata']['namespace'] == c['project']) + require(service['metadata'].get('uid') and service['metadata'].get('resourceVersion')) + require(service['spec']['traffic'] == [{'revisionName': c['warm_revision'], 'percent': 100}]) + require(warm['metadata']['name'] == c['warm_revision'] and warm['metadata'].get('uid')) + require(warm['metadata']['labels']['commit-sha'] == c['source_commit']) + require(str(warm['metadata']['annotations']['autoscaling.knative.dev/minScale']) == '1') + require(warm['status']['imageDigest'].split('@')[-1] == c['image_digest']) + require(minimum['name'] == 'projects/' + c['project'] + '/locations/' + c['region'] + + '/services/' + c['service']) + require(minimum['uid'] == service['metadata']['uid']) + require(minimum.get('scaling', {}).get('minInstanceCount', 0) == 0) + require(minimum.get('template', {}).get('scaling', {}).get('minInstanceCount', 0) == 0) + self.valid() + return dict(service_snapshot=service, warm=warm, service_v2=minimum) + + def verify_bucket(self): + value = self.api('bucket') + require(value['name'] == self.bucket and value['storageClass'] == 'STANDARD') + require(value['location'].lower() == self.config['region']) + require(value['iamConfiguration']['uniformBucketLevelAccess']['enabled'] is True) + require(value['iamConfiguration']['publicAccessPrevention'] == 'enforced') + require(value['lifecycle'] == LIFECYCLE) + require(not value.get('versioning', {}).get('enabled', False)) + require(not value.get('retentionPolicy') and not value.get('defaultEventBasedHold', False)) + require(int(value['softDeletePolicy']['retentionDurationSeconds']) == 0) + bindings = self.api('iam')['bindings'] + require(isinstance(bindings, list)) + require(not any(member in {'allUsers', 'allAuthenticatedUsers'} + for binding in bindings for member in binding['members'])) + self.valid() + + def upload(self, name, data): + self.valid() + name = self.prefix + name + self.api('upload', name=name, data=data) + require(self.api('download', name=name) == data) + self.valid() + + def execute(self): + c = self.config + initial = self.snapshot() + self.valid() + # Bucket creation is the distributed claim. A conflict/timeout is terminal; + # never adopt an existing bucket, repeat POST, delete, or repair its policy. + self.api('create_bucket', body={ + 'name': self.bucket, 'location': c['region'], 'storageClass': 'STANDARD', + 'iamConfiguration': {'uniformBucketLevelAccess': {'enabled': True}, + 'publicAccessPrevention': 'enforced'}, + 'lifecycle': LIFECYCLE, 'versioning': {'enabled': False}, + 'softDeletePolicy': {'retentionDurationSeconds': '0'}}) + self.verify_bucket() + require(self.snapshot() == initial) + self.upload('pre-warm-snapshot.json', encode({**initial, 'captured_at': self.now().isoformat()})) + require(self.snapshot() == initial) + card = {key: c[key] for key in ('authority', 'protected_environment', 'project', 'region', + 'service', 'source_commit', 'operator_commit', 'image_digest', 'request_id', + 'expires_at', 'restore_warm_on_failure')} + card.update(scope='s3-only', replacement=c['service'] + '-rec-' + c['request_id'], + service_snapshot=initial['service_snapshot'], warm=initial['warm']) + card_bytes = encode(card) + self.verify_bucket() + self.upload('recovery-card.json', card_bytes) + require(self.snapshot() == initial) + # Private receipt is not approval or a recover trigger. Public logs never + # contain locators, checksums, snapshots, configuration or provider errors. + self.upload('receipt.json', encode({ + 'card_uri': 'gs://' + self.bucket + '/' + self.prefix + 'recovery-card.json', + 'card_sha256': hashlib.sha256(card_bytes).hexdigest(), + 'operator_commit': c['operator_commit'], 'expires_at': c['expires_at'], + 'snapshot_stable': True, 'pricing_status': 'NOT_PROVEN', + 'recovery_dispatched': False})) + + +def main(): + try: + # Reuse the shared helper; register nested identity fields before work. + mask_identifiers({'IBKR_RECOVERY_BOOTSTRAP_JSON': os.environ['IBKR_RECOVERY_BOOTSTRAP_JSON']}) + config = json.loads(os.environ['IBKR_RECOVERY_BOOTSTRAP_JSON']) + Bootstrap(config, operator=os.environ['GITHUB_SHA'], + environment=os.environ['RECOVERY_PROTECTED_ENVIRONMENT']).execute() + except Exception: + print('Recovery evidence bootstrap failed; reconcile privately before any new request.', file=sys.stderr) + return 1 + print('Recovery evidence bootstrap completed; recovery remains separately gated.') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_recovery_evidence_bootstrap.py b/tests/test_recovery_evidence_bootstrap.py new file mode 100644 index 00000000..d49c5bfd --- /dev/null +++ b/tests/test_recovery_evidence_bootstrap.py @@ -0,0 +1,218 @@ +"""Offline bootstrap boundaries; all provider bindings are fake.""" +import copy +from datetime import datetime, timezone +import hashlib +import io +import json +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +import pytest + +from scripts.recovery_evidence_bootstrap import Bootstrap, BootstrapError, main + +NOW = datetime(2030, 1, 1, tzinfo=timezone.utc) + + +def config(): + return dict(authority='fake-authority', protected_environment='fake-protected', + project='fake-project', region='fake-region', service='fake-service', + warm_revision='fake-service-warm', request_id='fake1', bucket_suffix='fakeunique1', + source_commit='a' * 40, operator_commit='b' * 40, + image_digest='sha256:' + 'c' * 64, restore_warm_on_failure=True, + issued_at='2030-01-01T00:00:00+00:00', expires_at='2030-01-01T01:00:00+00:00') + + +class Fake: + def __init__(self): + self.service = {'metadata': {'name': 'fake-service', 'namespace': 'fake-project', + 'uid': 'fake-uid', 'resourceVersion': '1', 'generation': 1}, + 'spec': {'traffic': [{'revisionName': 'fake-service-warm', 'percent': 100}]}} + self.warm = {'metadata': {'name': 'fake-service-warm', 'uid': 'fake-warm-uid', + 'generation': 1, 'labels': {'commit-sha': 'a' * 40}, + 'annotations': {'autoscaling.knative.dev/minScale': '1'}}, + 'spec': {'containers': [{'resources': {'limits': {'cpu': '1', 'memory': '512Mi'}}}]}, + 'status': {'imageDigest': 'sha256:' + 'c' * 64}} + self.minimum = {'name': 'projects/fake-project/locations/fake-region/services/fake-service', + 'uid': 'fake-uid', 'scaling': {'minInstanceCount': 0}} + self.bucket = None + self.objects = {} + self.calls = [] + self.lock = Lock() + self.iam = {'bindings': []} + self.drift = False + self.reads = 0 + + def __call__(self, kind, **kwargs): + with self.lock: + self.calls.append(kind) + if kind == 'service': + self.reads += 1 + value = copy.deepcopy(self.service) + if self.drift and self.reads > 1: + value['metadata']['resourceVersion'] = 'changed' + return value + if kind == 'warm': return copy.deepcopy(self.warm) + if kind == 'minimum': return copy.deepcopy(self.minimum) + if kind == 'create_bucket': + if self.bucket is not None: raise BootstrapError('bootstrap failed') + self.bucket = copy.deepcopy(kwargs['body']) + return self.bucket + if kind == 'bucket': return copy.deepcopy(self.bucket) + if kind == 'iam': return self.iam + if kind == 'upload': + key = kwargs['name'] + if key in self.objects: raise BootstrapError('bootstrap failed') + self.objects[key] = kwargs['data'] + return {} + if kind == 'download': return self.objects[kwargs['name']] + raise AssertionError(kind) + + +def run(cloud=None, value=None, now=lambda: NOW): + cloud = cloud or Fake() + return Bootstrap(value or config(), api=cloud, now=now, + operator='b' * 40, environment='fake-protected').execute(), cloud + + +def test_private_card_contract_and_exact_object_readbacks(): + _, cloud = run() + card_bytes = cloud.objects['s3-recovery/fake1/recovery-card.json'] + card = json.loads(card_bytes) + assert card['scope'] == 's3-only' + assert card['warm'] == cloud.warm and card['service_snapshot'] == cloud.service + assert card['replacement'] == 'fake-service-rec-fake1' + receipt = json.loads(cloud.objects['s3-recovery/fake1/receipt.json']) + assert receipt['card_sha256'] == hashlib.sha256(card_bytes).hexdigest() + assert receipt['pricing_status'] == 'NOT_PROVEN' + assert set(cloud.calls) <= {'service', 'warm', 'minimum', 'create_bucket', 'bucket', + 'iam', 'upload', 'download'} + + +@pytest.mark.parametrize('field,value', [ + ('expires_at', '2029-01-01T00:00:00+00:00'), + ('expires_at', '2030-01-01T01:01:00+00:00'), + ('issued_at', '2030-01-01T00:01:00+00:00'), + ('operator_commit', 'd' * 40), ('protected_environment', ''), + ('restore_warm_on_failure', False), ('bucket_suffix', '../bad'), + ('request_id', 'a/b'), ('source_commit', 'main'), ('image_digest', '')]) +def test_invalid_authority_cannot_create_storage(field, value): + cloud, approval = Fake(), config() + approval[field] = value + with pytest.raises((BootstrapError, ValueError)): run(cloud, approval) + assert 'create_bucket' not in cloud.calls + + +def test_stale_service_does_not_publish_card(): + cloud = Fake() + cloud.drift = True + with pytest.raises(BootstrapError): run(cloud) + assert not cloud.objects + + +def test_concurrent_or_restarted_request_cannot_adopt_existing_bucket(): + cloud = Fake() + def attempt(): + try: + run(cloud) + return True + except BootstrapError: + return False + with ThreadPoolExecutor(max_workers=2) as pool: + assert sorted(pool.map(lambda _: attempt(), range(2))) == [False, True] + assert attempt() is False + assert cloud.calls.count('upload') == 3 + + +def test_unknown_bucket_create_outcome_is_not_retried(): + cloud = Fake() + def api(kind, **kwargs): + result = cloud(kind, **kwargs) + if kind == 'create_bucket': raise TimeoutError('fake private provider detail') + return result + with pytest.raises(TimeoutError): + Bootstrap(config(), api=api, now=lambda: NOW, operator='b' * 40, + environment='fake-protected').execute() + assert cloud.calls.count('create_bucket') == 1 + assert not cloud.objects + + +@pytest.mark.parametrize('case', ['public', 'lifecycle', 'uniform', 'readback', 'expiry']) +def test_integrity_or_expiry_failure_blocks_card(case): + cloud = Fake() + def api(kind, **kwargs): + result = cloud(kind, **kwargs) + if case == 'public' and kind == 'iam': + return {'bindings': [{'members': ['allUsers']}]} + if kind == 'bucket': + if case == 'lifecycle': result['lifecycle'] = {} + if case == 'uniform': result['iamConfiguration']['uniformBucketLevelAccess']['enabled'] = False + if case == 'readback' and kind == 'download': return b'corrupt' + return result + def now(): + return datetime(2031, 1, 1, tzinfo=timezone.utc) if case == 'expiry' and cloud.bucket else NOW + with pytest.raises(BootstrapError): + Bootstrap(config(), api=api, now=now, operator='b' * 40, + environment='fake-protected').execute() + assert 's3-recovery/fake1/recovery-card.json' not in cloud.objects + + +def test_cli_suppresses_private_provider_exception(monkeypatch, capsys): + monkeypatch.setenv('IBKR_RECOVERY_BOOTSTRAP_JSON', 'not json fake-private-detail') + assert main() == 1 + output = capsys.readouterr() + assert output.out == '' and 'private-detail' not in output.err + + +def test_shared_masker_handles_private_configuration(): + from scripts.mask_sync_identifiers import mask_identifiers + output = io.StringIO() + mask_identifiers({'IBKR_RECOVERY_BOOTSTRAP_JSON': json.dumps(config())}, output) + assert 'fake-project' in output.getvalue() + + +def test_transport_uses_create_only_preconditions_and_captures_diagnostics(monkeypatch): + from scripts import recovery_evidence_bootstrap as module + from unittest.mock import Mock + requests = [] + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def read(self): return b'{}' + def open_request(request, timeout): + requests.append(request) + assert timeout == 30 + return Response() + monkeypatch.setattr(module.subprocess, 'run', Mock(return_value=Mock(stdout='fake-token'))) + monkeypatch.setattr(module, 'urlopen', open_request) + api = module.API(config()) + api('upload', name='s3-recovery/fake1/recovery-card.json', data=b'{}') + assert requests[0].method == 'POST' + assert 'ifGenerationMatch=0' in requests[0].full_url + monkeypatch.setattr(module, 'urlopen', Mock(side_effect=RuntimeError('fake-private-detail'))) + with pytest.raises(BootstrapError) as error: api('bucket') + assert 'private-detail' not in str(error.value) + assert module.urlopen.call_count == 1 + + +def test_manual_protected_workflow_never_dispatches_recovery(): + from pathlib import Path + text = Path('.github/workflows/bootstrap-recovery-evidence.yml').read_text() + assert ' workflow_dispatch:\n' in text + assert 'run_attempt == 1' in text and "refs/heads/main" in text + assert 'environment: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }}' in text + assert 'group: ibkr-retained-warm-recovery' in text + assert 'cancel-in-progress: false' in text + assert ' contents: read\n id-token: write' in text + assert text.count('run:') == 1 + assert 'run: python -m scripts.recovery_evidence_bootstrap' in text + assert 'upload-artifact' not in text + assert 'pull_request' not in text and 'schedule:' not in text + + +def test_transport_refuses_redirect_with_credentials(): + from scripts.recovery_evidence_bootstrap import NoRedirect + from urllib.request import Request + with pytest.raises(BootstrapError): + NoRedirect().redirect_request(Request('https://example.invalid'), None, 302, + 'redirect', {}, 'https://other.invalid') From f3d4145829cf23c2fb81514fbc37b2fcddd00c39 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:28:15 +0800 Subject: [PATCH 2/3] fix: Require environment before cloud authentication --- .github/workflows/bootstrap-recovery-evidence.yml | 9 +++++++++ docs/ibkr-runtime-rollout.md | 3 ++- tests/test_recovery_evidence_bootstrap.py | 12 +++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bootstrap-recovery-evidence.yml b/.github/workflows/bootstrap-recovery-evidence.yml index 223e7a46..b757d3ec 100644 --- a/.github/workflows/bootstrap-recovery-evidence.yml +++ b/.github/workflows/bootstrap-recovery-evidence.yml @@ -25,6 +25,15 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.12" + - name: Verify protected environment binding + env: + RECOVERY_PROTECTED_ENVIRONMENT: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }} + run: | + python - <<'PYTHON' + import os + if not os.environ.get('RECOVERY_PROTECTED_ENVIRONMENT'): + raise SystemExit('Protected environment binding is required') + PYTHON - uses: google-github-actions/auth@v3 with: workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} diff --git a/docs/ibkr-runtime-rollout.md b/docs/ibkr-runtime-rollout.md index 6a309322..1cfe1c35 100644 --- a/docs/ibkr-runtime-rollout.md +++ b/docs/ibkr-runtime-rollout.md @@ -694,7 +694,8 @@ invokes recovery, changes Cloud Run, creates IAM/Secrets, or starts a warm revision. Merging its definition does not authorize dispatch. Native Environment approval and exact owner authority for the operator revision, target and storage scope remain required; review and validate that Environment's protection before -any later dispatch. Reruns are refused. +any later dispatch. A missing Environment name fails before credential acquisition. +Reruns are refused. An authorized operator supplies `IBKR_RECOVERY_BOOTSTRAP_JSON` as a protected Environment secret, never public dispatch inputs or repository Variables. It contains diff --git a/tests/test_recovery_evidence_bootstrap.py b/tests/test_recovery_evidence_bootstrap.py index d49c5bfd..a3e7c8cc 100644 --- a/tests/test_recovery_evidence_bootstrap.py +++ b/tests/test_recovery_evidence_bootstrap.py @@ -204,7 +204,7 @@ def test_manual_protected_workflow_never_dispatches_recovery(): assert 'group: ibkr-retained-warm-recovery' in text assert 'cancel-in-progress: false' in text assert ' contents: read\n id-token: write' in text - assert text.count('run:') == 1 + assert text.count('run:') == 2 assert 'run: python -m scripts.recovery_evidence_bootstrap' in text assert 'upload-artifact' not in text assert 'pull_request' not in text and 'schedule:' not in text @@ -216,3 +216,13 @@ def test_transport_refuses_redirect_with_credentials(): with pytest.raises(BootstrapError): NoRedirect().redirect_request(Request('https://example.invalid'), None, 302, 'redirect', {}, 'https://other.invalid') + + +def test_missing_environment_is_rejected_before_credential_acquisition(): + from pathlib import Path + text = Path('.github/workflows/bootstrap-recovery-evidence.yml').read_text() + guard = text.index('name: Verify protected environment binding') + auth = text.index('uses: google-github-actions/auth@v3') + assert guard < auth + assert "if not os.environ.get('RECOVERY_PROTECTED_ENVIRONMENT')" in text[guard:auth] + assert 'raise SystemExit' in text[guard:auth] From d89b06d85cfb24fafcd4556af6e751f398b80a59 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:55:56 +0800 Subject: [PATCH 3/3] fix: Validate bootstrap project identity and write expiry --- docs/ibkr-runtime-rollout.md | 9 ++- scripts/recovery_evidence_bootstrap.py | 19 ++++-- tests/test_recovery_evidence_bootstrap.py | 75 ++++++++++++++++++++++- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/docs/ibkr-runtime-rollout.md b/docs/ibkr-runtime-rollout.md index 1cfe1c35..0029bf75 100644 --- a/docs/ibkr-runtime-rollout.md +++ b/docs/ibkr-runtime-rollout.md @@ -709,7 +709,8 @@ letters/digits, the request identity follows the recovery-card grammar, and `restore_warm_on_failure` must be true. Keep these values private. Configuring these bindings is a separately authorized operation, not part of publishing code. -The existing Environment workload identity needs only `run.services.get`, +The existing Environment workload identity needs only `resourcemanager.projects.get`, +`run.services.get`, `run.revisions.get`, `storage.buckets.create`, `storage.buckets.get`, `storage.buckets.getIamPolicy`, `storage.objects.create`, and `storage.objects.get` for this entry. Scope storage permissions to the dedicated @@ -727,6 +728,12 @@ distributed single-use claim, including after runner restart. A conflict, timeou or uncertain write stops without retry, cleanup or a second suffix. Reconcile the original request privately before considering any separately approved new one. +Each snapshot resolves the configured project ID through Resource Manager and +validates its numeric project number. The v1 namespace must match that resolved +number or the configured ID; unknown or mismatched project bindings stop before +storage writes. Approval expiry is checked after acquiring each write token, +immediately before sending bucket or object creation. + Fresh v1 service, v1 warm revision and v2 service snapshots retain resource sizes, scaling, billing annotations and identity/configuration privately. Stable readbacks must match before a card is uploaded and before its receipt is retained. Only diff --git a/scripts/recovery_evidence_bootstrap.py b/scripts/recovery_evidence_bootstrap.py index 8019dc26..c16878a6 100644 --- a/scripts/recovery_evidence_bootstrap.py +++ b/scripts/recovery_evidence_bootstrap.py @@ -40,8 +40,8 @@ def encode(value): class API: """Single attempts with captured credentials/diagnostics; no SDK retries.""" - def __init__(self, config): - self.config = config + def __init__(self, config, *, now=lambda: datetime.now(timezone.utc)): + self.config, self.now = config, now self.bucket = 'ibkr-recovery-evidence-' + config['bucket_suffix'] def __call__(self, kind, *, body=None, name=None, data=None): @@ -51,6 +51,8 @@ def __call__(self, kind, *, body=None, name=None, data=None): bucket = 'https://storage.googleapis.com/storage/v1/b/' + self.bucket method = 'GET' if kind == 'service': url = base + '/services/' + c['service'] + elif kind == 'project': + url = 'https://cloudresourcemanager.googleapis.com/v1/projects/' + c['project'] elif kind == 'warm': url = base + '/revisions/' + c['warm_revision'] elif kind == 'minimum': url = ('https://run.googleapis.com/v2/projects/' + c['project'] @@ -73,6 +75,11 @@ def __call__(self, kind, *, body=None, name=None, data=None): request = Request(url, data=data, method=method, headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) + # Authentication can consume the remaining approval window. Recheck + # at the transport boundary, after token acquisition, for every POST. + if method == 'POST': + require(datetime.fromisoformat(c['issued_at']) <= self.now() + < datetime.fromisoformat(c['expires_at'])) with urlopen(request, timeout=30) as response: result = response.read() return result if kind == 'download' else json.loads(result) @@ -103,16 +110,20 @@ def __init__(self, config, *, operator, environment, api=None, self.valid() self.bucket = 'ibkr-recovery-evidence-' + c['bucket_suffix'] self.prefix = 's3-recovery/' + c['request_id'] + '/' - self.api = api or API(c) + self.api = api or API(c, now=now) def valid(self): require(self.issued <= self.now() < self.expires) def snapshot(self): c = self.config + project = self.api('project') + require(project['projectId'] == c['project']) + number = project['projectNumber'] + require(isinstance(number, str) and re.fullmatch(r'[1-9][0-9]*', number)) service, warm, minimum = (self.api(kind) for kind in ('service', 'warm', 'minimum')) require(service['metadata']['name'] == c['service']) - require(service['metadata']['namespace'] == c['project']) + require(service['metadata']['namespace'] in {c['project'], number}) require(service['metadata'].get('uid') and service['metadata'].get('resourceVersion')) require(service['spec']['traffic'] == [{'revisionName': c['warm_revision'], 'percent': 100}]) require(warm['metadata']['name'] == c['warm_revision'] and warm['metadata'].get('uid')) diff --git a/tests/test_recovery_evidence_bootstrap.py b/tests/test_recovery_evidence_bootstrap.py index a3e7c8cc..c2d8775d 100644 --- a/tests/test_recovery_evidence_bootstrap.py +++ b/tests/test_recovery_evidence_bootstrap.py @@ -35,6 +35,7 @@ def __init__(self): 'status': {'imageDigest': 'sha256:' + 'c' * 64}} self.minimum = {'name': 'projects/fake-project/locations/fake-region/services/fake-service', 'uid': 'fake-uid', 'scaling': {'minInstanceCount': 0}} + self.project = {'projectId': 'fake-project', 'projectNumber': '123456789012'} self.bucket = None self.objects = {} self.calls = [] @@ -52,6 +53,7 @@ def __call__(self, kind, **kwargs): if self.drift and self.reads > 1: value['metadata']['resourceVersion'] = 'changed' return value + if kind == 'project': return copy.deepcopy(self.project) if kind == 'warm': return copy.deepcopy(self.warm) if kind == 'minimum': return copy.deepcopy(self.minimum) if kind == 'create_bucket': @@ -86,7 +88,7 @@ def test_private_card_contract_and_exact_object_readbacks(): assert receipt['card_sha256'] == hashlib.sha256(card_bytes).hexdigest() assert receipt['pricing_status'] == 'NOT_PROVEN' assert set(cloud.calls) <= {'service', 'warm', 'minimum', 'create_bucket', 'bucket', - 'iam', 'upload', 'download'} + 'iam', 'upload', 'download', 'project'} @pytest.mark.parametrize('field,value', [ @@ -185,7 +187,7 @@ def open_request(request, timeout): return Response() monkeypatch.setattr(module.subprocess, 'run', Mock(return_value=Mock(stdout='fake-token'))) monkeypatch.setattr(module, 'urlopen', open_request) - api = module.API(config()) + api = module.API(config(), now=lambda: NOW) api('upload', name='s3-recovery/fake1/recovery-card.json', data=b'{}') assert requests[0].method == 'POST' assert 'ifGenerationMatch=0' in requests[0].full_url @@ -226,3 +228,72 @@ def test_missing_environment_is_rejected_before_credential_acquisition(): assert guard < auth assert "if not os.environ.get('RECOVERY_PROTECTED_ENVIRONMENT')" in text[guard:auth] assert 'raise SystemExit' in text[guard:auth] + + +@pytest.mark.parametrize('namespace', ['fake-project', '123456789012']) +def test_resolved_project_namespace_accepts_id_and_number(namespace): + cloud = Fake() + cloud.service['metadata']['namespace'] = namespace + run(cloud) + assert cloud.calls.index('project') < cloud.calls.index('create_bucket') + + +@pytest.mark.parametrize('project', [ + {'projectId': 'other-fake-project', 'projectNumber': '123456789012'}, + {'projectId': 'fake-project', 'projectNumber': ''}, + {'projectId': 'fake-project', 'projectNumber': None}, + {'projectId': 'fake-project', 'projectNumber': 'invalid'}, +]) +def test_invalid_project_resolution_blocks_writes(project): + cloud = Fake() + cloud.project = project + with pytest.raises(BootstrapError): run(cloud) + assert 'create_bucket' not in cloud.calls + + +def test_foreign_numeric_namespace_blocks_writes(): + cloud = Fake() + cloud.service['metadata']['namespace'] = '999999999999' + with pytest.raises(BootstrapError): run(cloud) + assert 'create_bucket' not in cloud.calls + + +@pytest.mark.parametrize('kind', ['create_bucket', 'upload']) +def test_expiry_during_credential_acquisition_blocks_transport_write(monkeypatch, kind): + from scripts import recovery_evidence_bootstrap as module + from unittest.mock import Mock + clock = [NOW] + def acquire(*args, **kwargs): + clock[0] = datetime(2030, 1, 1, 1, tzinfo=timezone.utc) + return Mock(stdout='fake-token') + monkeypatch.setattr(module.subprocess, 'run', acquire) + request = Mock() + monkeypatch.setattr(module, 'urlopen', request) + # Exercise the real transport from Bootstrap, including its injected clock. + bootstrap = Bootstrap(config(), now=lambda: clock[0], operator='b' * 40, + environment='fake-protected') + bootstrap.valid() + with pytest.raises(BootstrapError): + bootstrap.api(kind, body={}, name='fake-object', data=b'{}') + request.assert_not_called() + + +def test_project_lookup_transport_and_private_error(monkeypatch): + from scripts import recovery_evidence_bootstrap as module + from unittest.mock import Mock + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def read(self): + return b'{"projectId":"fake-project","projectNumber":"123456789012"}' + monkeypatch.setattr(module.subprocess, 'run', Mock(return_value=Mock(stdout='fake-token'))) + request = Mock(return_value=Response()) + monkeypatch.setattr(module, 'urlopen', request) + api = module.API(config(), now=lambda: NOW) + assert api('project')['projectNumber'] == '123456789012' + sent = request.call_args.args[0] + assert sent.method == 'GET' + assert sent.full_url == 'https://cloudresourcemanager.googleapis.com/v1/projects/fake-project' + request.side_effect = RuntimeError('fake-private-project-detail') + with pytest.raises(BootstrapError) as error: api('project') + assert 'private-project-detail' not in str(error.value)