diff --git a/.github/workflows/read-warm-recovery-snapshot.yml b/.github/workflows/read-warm-recovery-snapshot.yml new file mode 100644 index 00000000..c8859a43 --- /dev/null +++ b/.github/workflows/read-warm-recovery-snapshot.yml @@ -0,0 +1,66 @@ +name: Read private warm recovery snapshot + +on: + workflow_dispatch: + inputs: + mode: + description: "Read current warm configuration or verify bootstrap receipt" + type: choice + options: + - snapshot + - receipt + required: true + +permissions: + contents: read + id-token: write + +concurrency: + group: ibkr-retained-warm-recovery + cancel-in-progress: false + +jobs: + read: + if: github.event_name == 'workflow_dispatch' && github.run_attempt == 1 && github.ref == 'refs/heads/main' + environment: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Verify protected read bindings + env: + RECOVERY_PROTECTED_ENVIRONMENT: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }} + IBKR_RECOVERY_READER_JSON: ${{ secrets.IBKR_RECOVERY_READER_JSON }} + IBKR_RECOVERY_READER_CERT_PEM: ${{ secrets.IBKR_RECOVERY_READER_CERT_PEM }} + run: | + python - <<'PY' + import os + if not all(os.environ.get(key) for key in ( + 'RECOVERY_PROTECTED_ENVIRONMENT', 'IBKR_RECOVERY_READER_JSON', + 'IBKR_RECOVERY_READER_CERT_PEM')): + raise SystemExit('Protected read bindings are required') + PY + - uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@v3 + - name: Read and encrypt private evidence + env: + RECOVERY_PROTECTED_ENVIRONMENT: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }} + IBKR_RECOVERY_READER_JSON: ${{ secrets.IBKR_RECOVERY_READER_JSON }} + IBKR_RECOVERY_READER_CERT_PEM: ${{ secrets.IBKR_RECOVERY_READER_CERT_PEM }} + READER_MODE: ${{ inputs.mode }} + run: python -m scripts.warm_snapshot_reader + - uses: actions/upload-artifact@v4 + with: + name: encrypted-warm-readback + path: ${{ runner.temp }}/warm-reader/warm-readback.cms + if-no-files-found: error + retention-days: 1 diff --git a/docs/ibkr-runtime-rollout.md b/docs/ibkr-runtime-rollout.md index 0029bf75..f2a825ba 100644 --- a/docs/ibkr-runtime-rollout.md +++ b/docs/ibkr-runtime-rollout.md @@ -752,3 +752,65 @@ receipt and card checksum, fresh target state and native Environment approval. Expiry stops bootstrap writes; this entry has no compensation because it never changes warm/traffic state. It cannot extend a prior approval or automatically launch recovery after merge. + + +### Encrypted read-only warm snapshot and receipt + +`read-warm-recovery-snapshot.yml` is a separate manual, main-only reader in the +existing protected Environment and retained-warm recovery concurrency group. +Only its definition is being introduced. Publication or merge does not authorize +configuration, dispatch, bootstrap, recovery, or a change of recovery operator. +The retained bootstrap/recovery operator remains the previously approved merged +operator, supplied privately in `operator_commit`; the reader's checkout is not +an operator rebinding. Native Environment reviewer protection must be read back +before any separately authorized run. Missing protection is an operational blocker. + +Two protected Environment secrets are prerequisites, provisioned only under a +separate authorization: + +- `IBKR_RECOVERY_READER_JSON`: `protected_environment`, `project`, `region`, + `service`, and `operator_commit`; receipt mode additionally requires the exact + bootstrap `bucket_suffix` and `request_id`. Keep bindings out of dispatch inputs. +- `IBKR_RECOVERY_READER_CERT_PEM`: the coordinator's trusted public X.509 encryption + certificate. Keep its private key solely in the approved coordinator's private + execution channel, never in Actions, repository secrets, Git, or this box. + Verify recipient ownership before configuring the certificate. Missing or + unusable certificates fail without an export; there is no plaintext fallback. + +The workflow reuses existing WIF credentials. The reader sends only bounded GET +requests for the configured Resource Manager project, Cloud Run v1 service and +retained revision, Cloud Run v2 service, and (in receipt mode) the three exact +bootstrap objects. It never lists accounts, reads Secret Manager, writes storage, +changes IAM or Cloud Run, or invokes another workflow. Permissions needed are +`resourcemanager.projects.get`, `run.services.get`, `run.revisions.get` and, for +receipt mode, `storage.objects.get` on the exact approved bootstrap prefix. +The existing identity may have broader capabilities; this code has no cloud write +transport. Missing permissions fail rather than trigger a grant or identity change. + +Snapshot mode exports current `project`, `region`, `service`, `warm_revision`, +`source_commit`, and `image_digest`, with private v1/v2 snapshots for scaling, +billing and configuration assessment. Both reads must match. Split/tagged traffic, +foreign namespaces, unexpected minimums and malformed provenance fail closed. +Authority, request identity, bucket suffix, time window and restoration decision +are owner-supplied bootstrap inputs, not facts discoverable from Cloud Run; this +reader neither invents them nor renews a TTL. Receipt mode additionally verifies +the exact receipt/card checksum, retained operator, request/target bindings and +stored/current snapshot agreement, then re-reads all three objects and current +state. Missing, changed, or invalid evidence fails. A receipt may describe an +expired approval; `READ_VERIFIED` describes evidence integrity only and is never +permission to recover. Evidence is time-bounded, not an atomic cloud transaction. + +The script encrypts its in-memory JSON using OpenSSL authenticated CMS with +AES-256-GCM. It writes no plaintext evidence file and emits fixed diagnostics. +Only `warm-readback.cms` is uploaded, on success, as `encrypted-warm-readback` +with one-day requested artifact retention. The repository is public: artifact +access controls are not a privacy guarantee. Only possession of the coordinator's +private key enables decryption; encrypted artifacts must never be mistaken for +public-safe plaintext evidence. GitHub artifact retention does not revoke copies. +The coordinator downloads and decrypts only in its approved private channel, +checks the producer run/head and artifact digest, and assembles any later +bootstrap configuration privately. No plaintext, object locator or evidence +checksum belongs in an issue, PR, public log or job summary. A rerun or preexisting +local export directory is refused; concurrent approved reads cannot overwrite +one another. An interrupted/failed run uploads nothing. Pricing, recovery and +runtime qualification remain unproved and separately gated. diff --git a/scripts/warm_snapshot_reader.py b/scripts/warm_snapshot_reader.py new file mode 100644 index 00000000..d78ceae9 --- /dev/null +++ b/scripts/warm_snapshot_reader.py @@ -0,0 +1,169 @@ +"""GET-only warm configuration and receipt reader; export ciphertext only.""" +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +from urllib.parse import quote +from urllib.request import Request + +from scripts.mask_sync_identifiers import mask_identifiers +from scripts.recovery_evidence_bootstrap import encode, require, urlopen + + +class Reader: + def __init__(self, config, *, environment, get=None): + self.config = config + for key in ('project', 'region', 'service'): + require(isinstance(config[key], str) + and re.fullmatch(r'[a-z][a-z0-9-]{0,62}', config[key])) + require(environment and config['protected_environment'] == environment) + require(re.fullmatch(r'[a-f0-9]{40}', config['operator_commit'])) + self.get = get or self.request + + def request(self, url): + # Fixed, internally constructed endpoints only. No write method, body, + # redirect, automatic retry or provider diagnostics can escape here. + token = subprocess.run(['gcloud', 'auth', 'print-access-token'], + capture_output=True, check=True, text=True, + timeout=15).stdout.strip() + require(token) + request = Request(url, method='GET', headers={'Authorization': 'Bearer ' + token}) + with urlopen(request, timeout=30) as response: + data = response.read(4 * 1024 * 1024 + 1) + require(len(data) <= 4 * 1024 * 1024) + return data + + def snapshot(self): + c = self.config + base = ('https://' + c['region'] + '-run.googleapis.com/apis/serving.knative.dev/v1/' + 'namespaces/' + c['project']) + project = json.loads(self.get('https://cloudresourcemanager.googleapis.com/v1/projects/' + + c['project'])) + require(project['projectId'] == c['project']) + number = project['projectNumber'] + require(isinstance(number, str) and re.fullmatch(r'[1-9][0-9]*', number)) + service = json.loads(self.get(base + '/services/' + c['service'])) + require(service['metadata']['name'] == c['service']) + require(service['metadata']['namespace'] in {c['project'], number}) + require(service['metadata'].get('uid') and service['metadata'].get('resourceVersion')) + traffic = service['spec']['traffic'] + require(isinstance(traffic, list) and len(traffic) == 1) + revision = traffic[0]['revisionName'] + require(isinstance(revision, str) and re.fullmatch(r'[a-z][a-z0-9-]{0,62}', revision)) + require(revision.startswith(c['service'] + '-')) + require(traffic == [{'revisionName': revision, 'percent': 100}]) + warm = json.loads(self.get(base + '/revisions/' + revision)) + require(warm['metadata']['name'] == revision and warm['metadata'].get('uid')) + require(warm['metadata']['namespace'] in {c['project'], number}) + source = warm['metadata']['labels']['commit-sha'] + digest = warm['status']['imageDigest'].split('@')[-1] + require(re.fullmatch(r'[a-f0-9]{40}', source)) + require(re.fullmatch(r'sha256:[a-f0-9]{64}', digest)) + require(str(warm['metadata']['annotations']['autoscaling.knative.dev/minScale']) == '1') + minimum = json.loads(self.get('https://run.googleapis.com/v2/projects/' + c['project'] + + '/locations/' + c['region'] + '/services/' + c['service'])) + require(minimum['name'] == 'projects/' + c['project'] + '/locations/' + c['region'] + + '/services/' + c['service']) + require(minimum['uid'] == service['metadata']['uid']) + require(minimum.get('scaling', {}).get('minInstanceCount', 0) == 0) + require(minimum.get('template', {}).get('scaling', {}).get('minInstanceCount', 0) == 0) + return dict(project=c['project'], region=c['region'], service=c['service'], + warm_revision=revision, source_commit=source, image_digest=digest, + service_snapshot=service, warm=warm, service_v2=minimum) + + def receipt(self, snapshot): + c = self.config + require(re.fullmatch(r'[a-z0-9]{8,32}', c['bucket_suffix'])) + require(re.fullmatch(r'[a-z0-9][a-z0-9-]{0,30}', c['request_id'])) + bucket = 'ibkr-recovery-evidence-' + c['bucket_suffix'] + prefix = 's3-recovery/' + c['request_id'] + '/' + def read(name): + return self.get('https://storage.googleapis.com/storage/v1/b/' + bucket + + '/o/' + quote(prefix + name, safe='') + '?alt=media') + receipt_bytes, card_bytes, before_bytes = (read(name) for name in + ('receipt.json', 'recovery-card.json', 'pre-warm-snapshot.json')) + receipt, card, before = map(json.loads, (receipt_bytes, card_bytes, before_bytes)) + require(receipt['card_uri'] == 'gs://' + bucket + '/' + prefix + 'recovery-card.json') + require(receipt['card_sha256'] == hashlib.sha256(card_bytes).hexdigest()) + require(receipt['operator_commit'] == card['operator_commit'] == c['operator_commit']) + require(receipt['snapshot_stable'] is True and receipt['recovery_dispatched'] is False) + require(receipt['pricing_status'] == 'NOT_PROVEN') + require(receipt['expires_at'] == card['expires_at']) + require(card['scope'] == 's3-only' and card['restore_warm_on_failure'] is True) + require(card['protected_environment'] == c['protected_environment'] and card['authority']) + require(card['request_id'] == c['request_id']) + require(card['replacement'] == c['service'] + '-rec-' + c['request_id']) + for key in ('project', 'region', 'service', 'source_commit', 'image_digest', + 'service_snapshot', 'warm'): + require(card[key] == snapshot[key]) + for key in ('service_snapshot', 'warm', 'service_v2'): + require(before[key] == snapshot[key]) + require(read('receipt.json') == receipt_bytes) + require(read('recovery-card.json') == card_bytes) + require(read('pre-warm-snapshot.json') == before_bytes) + # Expiry is retained as evidence, never extended or interpreted as approval. + return dict(receipt=receipt, card=card, pre_warm_snapshot=before) + + def collect(self, mode): + require(mode in {'snapshot', 'receipt'}) + initial = self.snapshot() + private = self.receipt(initial) if mode == 'receipt' else None + require(self.snapshot() == initial) + return dict(schema_version=1, captured_at=datetime.now(timezone.utc).isoformat(), + operator_commit=self.config['operator_commit'], snapshot=initial, + bootstrap_receipt=private, + receipt_status='READ_VERIFIED' if private else 'NOT_REQUESTED', + authority_status='NOT_EVALUATED', recovery_dispatched=False) + + +def encrypt(payload, certificate, root): + # No plaintext output file. Authenticated CMS encryption uses a coordinator + # certificate; the corresponding private key must never enter Actions. + cert = root / 'coordinator.pem' + output = root / 'warm-readback.cms' + with cert.open('x', encoding='ascii') as file: + file.write(certificate) + try: + result = subprocess.run(['openssl', 'cms', '-encrypt', '-aes-256-gcm', + '-binary', '-outform', 'DER', str(cert)], + input=encode(payload), capture_output=True, check=True, timeout=30) + require(result.stdout) + with output.open('xb') as file: + file.write(result.stdout) + finally: + cert.unlink() + + +def main(): + os.umask(0o077) + try: + raw = os.environ['IBKR_RECOVERY_READER_JSON'] + # Shared helper is the only masking implementation; no payload is printed. + mask_identifiers({'IBKR_RECOVERY_READER_JSON': raw}) + config = json.loads(raw) + certificate = os.environ['IBKR_RECOVERY_READER_CERT_PEM'] + require(certificate and 'PRIVATE KEY' not in certificate) + root = Path(os.environ['RUNNER_TEMP']) / 'warm-reader' + # Exclusive directory creation refuses a restarted/stale export directory. + root.mkdir(mode=0o700) + # Validate certificate/encryption before acquiring cloud credentials. + encrypt({'probe': True}, certificate, root) + (root / 'warm-readback.cms').unlink() + result = Reader(config, environment=os.environ['RECOVERY_PROTECTED_ENVIRONMENT']).collect( + os.environ['READER_MODE']) + encrypt(result, certificate, root) + except Exception: + print('Private warm readback failed; no recovery is authorized.', file=sys.stderr) + return 1 + print('Encrypted readback ready; recovery remains separately gated.') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_warm_snapshot_reader.py b/tests/test_warm_snapshot_reader.py new file mode 100644 index 00000000..2fcce3f5 --- /dev/null +++ b/tests/test_warm_snapshot_reader.py @@ -0,0 +1,194 @@ +"""Offline read-only transport, drift and encrypted export boundaries.""" +import json +import os +from pathlib import Path +import subprocess +from unittest.mock import Mock + +import pytest + +from scripts import warm_snapshot_reader as module +from scripts.recovery_evidence_bootstrap import BootstrapError +from tests.test_recovery_evidence_bootstrap import Fake, config + + +def reader(cloud=None): + cloud = cloud or Fake() + cloud.warm['metadata']['namespace'] = 'fake-project' + calls = [] + def get(url): + calls.append(url) + if '/revisions/' in url: value = cloud('warm') + elif '/v2/' in url: value = cloud('minimum') + elif '/services/' in url: value = cloud('service') + elif 'cloudresourcemanager' in url: value = cloud('project') + else: + from urllib.parse import unquote + return cloud.objects[unquote(url.split('/o/')[1].split('?')[0])] + return json.dumps(value).encode() + return module.Reader(config(), environment='fake-protected', get=get), cloud, calls + + +def test_missing_receipt_fails_instead_of_exporting_success(): + instance, _, _ = reader() + with pytest.raises(KeyError): instance.collect('receipt') + + +def test_changed_snapshot_blocks_export(): + instance, cloud, _ = reader() + cloud.drift = True + with pytest.raises(BootstrapError): instance.collect('snapshot') + + +def test_readonly_export_has_all_live_bootstrap_fields(): + instance, _, calls = reader() + value = instance.collect('snapshot') + assert set(('project', 'region', 'service', 'warm_revision', 'source_commit', 'image_digest')) <= value['snapshot'].keys() + assert value['receipt_status'] == 'NOT_REQUESTED' + assert value['recovery_dispatched'] is False + assert len(calls) == 8 + assert all(url.startswith('https://') for url in calls) + + +@pytest.mark.parametrize('case', ['namespace', 'digest', 'traffic', 'minimum', 'project']) +def test_invalid_live_state_is_rejected(case): + instance, cloud, _ = reader() + if case == 'namespace': cloud.warm['metadata']['namespace'] = 'fake-foreign' + if case == 'digest': cloud.warm['status']['imageDigest'] = 'invalid' + if case == 'traffic': cloud.service['spec']['traffic'].append({'latestRevision': True, 'percent': 0}) + if case == 'minimum': cloud.minimum['scaling']['minInstanceCount'] = 1 + if case == 'project': cloud.project['projectId'] = 'fake-foreign' + with pytest.raises(BootstrapError): instance.collect('snapshot') + + +def test_transport_can_only_send_get_and_captures_token(monkeypatch): + instance, _, _ = reader() + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.read.return_value = b'{}' + open_request = Mock(return_value=response) + monkeypatch.setattr(module, 'urlopen', open_request) + monkeypatch.setattr(module.subprocess, 'run', Mock(return_value=Mock(stdout='fake-token'))) + assert instance.request('https://example.invalid') == b'{}' + sent = open_request.call_args.args[0] + assert sent.method == 'GET' and sent.data is None + assert module.subprocess.run.call_args.kwargs['capture_output'] is True + + +def test_receipt_hash_and_operator_binding(): + from scripts.recovery_evidence_bootstrap import Bootstrap + from tests.test_recovery_evidence_bootstrap import NOW + instance, cloud, _ = reader() + Bootstrap(config(), operator='b' * 40, environment='fake-protected', + api=cloud, now=lambda: NOW).execute() + assert instance.collect('receipt')['receipt_status'] == 'READ_VERIFIED' + name = 's3-recovery/fake1/recovery-card.json' + original = cloud.objects[name] + cloud.objects[name] = original + b' ' + with pytest.raises(BootstrapError): instance.collect('receipt') + cloud.objects[name] = original + instance.config = {**config(), 'operator_commit': 'd' * 40} + with pytest.raises(BootstrapError): instance.collect('receipt') + + +def test_invalid_private_configuration_never_leaks(monkeypatch, capsys, tmp_path): + monkeypatch.setenv('IBKR_RECOVERY_READER_JSON', 'fake-private-malformed') + monkeypatch.setenv('RUNNER_TEMP', str(tmp_path)) + assert module.main() == 1 + output = capsys.readouterr() + assert 'fake-private-malformed' not in output.out + output.err + assert not list(tmp_path.rglob('*.cms')) + + +def test_cms_roundtrip_and_exclusive_export(tmp_path): + key, cert = tmp_path / 'key.pem', tmp_path / 'cert.pem' + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:3072', '-nodes', + '-subj', '/CN=fake-coordinator', '-keyout', str(key), '-out', str(cert), + '-days', '1'], check=True, capture_output=True) + certificate = cert.read_text() + cert.unlink() + payload = {'private': 'fake-private-readback'} + previous = os.umask(0o077) + try: + module.encrypt(payload, certificate, tmp_path) + finally: + os.umask(previous) + encrypted = tmp_path / 'warm-readback.cms' + assert b'fake-private-readback' not in encrypted.read_bytes() + assert encrypted.stat().st_mode & 0o777 == 0o600 + decrypted = subprocess.run(['openssl', 'cms', '-decrypt', '-inform', 'DER', + '-inkey', str(key), '-in', str(encrypted)], + capture_output=True, check=True).stdout + assert json.loads(decrypted) == payload + with pytest.raises(FileExistsError): module.encrypt(payload, certificate, tmp_path) + assert not (tmp_path / 'coordinator.pem').exists() + damaged = bytearray(encrypted.read_bytes()) + damaged[-1] ^= 1 + encrypted.write_bytes(damaged) + assert subprocess.run(['openssl', 'cms', '-decrypt', '-inform', 'DER', + '-inkey', str(key), '-in', str(encrypted)], capture_output=True).returncode != 0 + + +def test_workflow_only_exports_ciphertext_after_success(): + text = Path('.github/workflows/read-warm-recovery-snapshot.yml').read_text() + assert 'refs/heads/main' in text and 'run_attempt == 1' in text + assert 'environment: ${{ vars.IBKR_DEPLOY_ENVIRONMENT }}' in text + assert 'group: ibkr-retained-warm-recovery' in text + assert 'cancel-in-progress: false' in text + assert 'warm-reader/warm-readback.cms' in text + assert 'if-no-files-found: error' in text + assert 'always()' not in text + assert 'pull_request:' not in text and 'schedule:' not in text + assert 'IBKR_RECOVERY_BOOTSTRAP_JSON' not in text + assert 'cloud_run_warm_recovery' not in text + + +def test_restarted_export_directory_fails_closed(monkeypatch, tmp_path, capsys): + monkeypatch.setenv('IBKR_RECOVERY_READER_JSON', json.dumps(config())) + monkeypatch.setenv('IBKR_RECOVERY_READER_CERT_PEM', 'fake-certificate') + monkeypatch.setenv('RECOVERY_PROTECTED_ENVIRONMENT', 'fake-protected') + monkeypatch.setenv('READER_MODE', 'snapshot') + monkeypatch.setenv('RUNNER_TEMP', str(tmp_path)) + (tmp_path / 'warm-reader').mkdir() + transport = Mock() + monkeypatch.setattr(module.Reader, 'request', transport) + assert module.main() == 1 + transport.assert_not_called() + assert not list(tmp_path.rglob('*.cms')) + assert 'fake-certificate' not in capsys.readouterr().err + + +def test_unknown_provider_failure_has_no_retry_or_private_diagnostics(monkeypatch, capsys, tmp_path): + monkeypatch.setenv('IBKR_RECOVERY_READER_JSON', json.dumps(config())) + monkeypatch.setenv('IBKR_RECOVERY_READER_CERT_PEM', 'fake-certificate') + monkeypatch.setenv('RECOVERY_PROTECTED_ENVIRONMENT', 'fake-protected') + monkeypatch.setenv('READER_MODE', 'snapshot') + monkeypatch.setenv('RUNNER_TEMP', str(tmp_path)) + def encrypt(payload, cert, root): + (root / 'warm-readback.cms').write_bytes(b'fake-encrypted-probe') + monkeypatch.setattr(module, 'encrypt', encrypt) + transport = Mock(side_effect=TimeoutError('fake-private-provider-detail')) + monkeypatch.setattr(module.Reader, 'request', transport) + assert module.main() == 1 + assert transport.call_count == 1 + assert 'fake-private-provider-detail' not in capsys.readouterr().err + assert not list(tmp_path.rglob('*.cms')) + + +def test_receipt_changing_between_reads_blocks_export(): + from scripts.recovery_evidence_bootstrap import Bootstrap + from tests.test_recovery_evidence_bootstrap import NOW + instance, cloud, _ = reader() + Bootstrap(config(), operator='b' * 40, environment='fake-protected', + api=cloud, now=lambda: NOW).execute() + original = instance.get + count = [0] + def get(url): + result = original(url) + if 'receipt.json' in url: + count[0] += 1 + if count[0] > 1: return result + b' ' + return result + instance.get = get + with pytest.raises(BootstrapError): instance.collect('receipt')