From 6d55d816f3ce72597654534d1ec040a40094d550 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Thu, 22 Oct 2026 06:51:39 +0000 Subject: [PATCH 1/7] Add a per-request admission guard to the scan worker The sandboxed scanner holds no key, so before every connection to the customer's site it now asks the root supervisor over a private socket pair. The supervisor calls a new signed `admit` route; WordPress decides under the same queue lock that serializes customer Stop, operator revocation, expiry, claims, starts and results. It checks job state, lease, recipient confirmation, saved authorization, the frozen profile, one-per-second pacing and the request budget, and records the reservation in the same compare-and-set update as the decision. A grant is consumed when issued, is valid for one second and covers one connection. The supervisor kills the scanner on any refusal, protocol violation, or request running past the disclosed ten-second bound, and probes liveness every two seconds (including during backoff) so a Stop aborts the one request already admitted. SubprocessRunner.run refuses to start without a gate, and a scanner without the supervisor socket refuses to run. Unreachable guards, malformed answers and unknown profiles are refusals. Schema 3 adds profile_id, http_admitted and last_admit_ms (existing rows default to the r2-header-v1 profile with empty counters). Customer Stop and operator revocation now wait up to three seconds for the queue lock so a concurrent admission cannot make them fail. Scanning remains disabled by default; the profile limits are code constants, not configuration. Refs CompleteTech-LLC/complete.tech#17 (step 1, part 2). Co-Authored-By: Claude Code --- .github/workflows/tests.yml | 4 +- deployment/test-http-worker.py | 51 +- docs/WORDPRESS.md | 79 +- integrations/wordpress/ct-scan.php | 175 +++- .../wordpress/tests/admission-disabled.php | 7 + integrations/wordpress/tests/admission.php | 220 +++++ integrations/wordpress/tests/integration.php | 99 +-- integrations/wordpress/tests/lock-holder.php | 12 + integrations/wordpress/tests/support.php | 135 +++ integrations/worker/admission.py | 339 +++++++ integrations/worker/network.py | 67 +- integrations/worker/protocol.py | 40 +- integrations/worker/runner.py | 178 ++-- integrations/worker/scanner.py | 24 +- integrations/worker/service.py | 4 +- test_admission.py | 833 ++++++++++++++++++ test_scan_worker.py | 52 +- 17 files changed, 2087 insertions(+), 232 deletions(-) create mode 100644 integrations/wordpress/tests/admission-disabled.php create mode 100644 integrations/wordpress/tests/admission.php create mode 100644 integrations/wordpress/tests/lock-holder.php create mode 100644 integrations/wordpress/tests/support.php create mode 100644 integrations/worker/admission.py create mode 100644 test_admission.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e56072..2c7aa8e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,6 +47,8 @@ jobs: run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/deployment/test-bootstrap.php - name: Test queue, confirmation and receipts run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/integrations/wordpress/tests/integration.php + - name: Test per-request admission, cancellation race and fail-closed behavior + run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/integrations/wordpress/tests/admission.php - name: Test Python to WordPress over HTTP with synthetic scan responses run: | fixture=$(docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/deployment/test-http-fixture.php setup) @@ -60,7 +62,7 @@ jobs: --tmpfs /tmp:rw,noexec,nosuid,size=32m,mode=1777 -v "$PWD:/app:ro" -w /app -e PYTHONDONTWRITEBYTECODE=1 python:3.12-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 - python -m unittest -v test_scan_worker + python -m unittest -v test_scan_worker test_admission - name: Build and check packaged isolation run: | docker build --build-arg CHECKER_REVISION="$GITHUB_SHA" -f deployment/Dockerfile -t ct-worker-fixture . diff --git a/deployment/test-http-worker.py b/deployment/test-http-worker.py index b3278f2..ffcac23 100644 --- a/deployment/test-http-worker.py +++ b/deployment/test-http-worker.py @@ -14,12 +14,18 @@ from email.message import Message from urllib.parse import urlsplit, urlencode +from integrations.worker.admission import AdmissionDenied, AdmissionGate, parse_grant from integrations.worker.protocol import APIClient, APIError from integrations.worker.runner import SubprocessRunner from integrations.worker.network import RequestOrigin from wordpress_check import Checker +# The disposable Compose service is "wordpress:80"; a local run may override both. +FIXTURE_HOST = os.environ.get('CT_FIXTURE_HOST', 'wordpress') +FIXTURE_PORT = int(os.environ.get('CT_FIXTURE_PORT', '80')) + + class Response: def __init__(self, response, connection): self.response = response @@ -39,8 +45,8 @@ def __exit__(self, *_): class FixtureTransport: def open(self, request, timeout): route = urlsplit(request.full_url).path.removeprefix('/wp-json') - assert route in {'/ct-scan/v1/claim', '/ct-scan/v1/start', '/ct-scan/v1/result'} - connection = http.client.HTTPConnection('wordpress', 80, timeout=timeout) + assert route in {'/ct-scan/v1/claim', '/ct-scan/v1/start', '/ct-scan/v1/admit', '/ct-scan/v1/result'} + connection = http.client.HTTPConnection(FIXTURE_HOST, FIXTURE_PORT, timeout=timeout) connection.request('POST', '/?rest_route=' + route, body=request.data, headers=dict(request.header_items())) response = connection.getresponse() return Response(response, connection) @@ -74,7 +80,7 @@ def open(self, request, timeout): token = os.environ.get('CT_FIXTURE_TOKEN') if token: assert re.fullmatch('[a-f0-9]{64}', token) - connection = http.client.HTTPConnection('wordpress', timeout=10) + connection = http.client.HTTPConnection(FIXTURE_HOST, FIXTURE_PORT, timeout=10) connection.request('POST', '/?rest_route=/ct-scan/v1/claim', '{}', {'Content-Type': 'application/json'}) response = connection.getresponse() response.read() @@ -89,7 +95,7 @@ def open(self, request, timeout): assert error.status == 409 else: raise AssertionError('Replayed signed request accepted') - connection = http.client.HTTPConnection('wordpress', timeout=10) + connection = http.client.HTTPConnection(FIXTURE_HOST, FIXTURE_PORT, timeout=10) connection.request('GET', '/wp-admin/admin-post.php?action=ct_scan_confirm&token=' + token) response = connection.getresponse() page = response.read().decode() @@ -100,7 +106,7 @@ def open(self, request, timeout): fields = dict(re.findall(r'name="([^"]+)" value="([^"]*)"', page)) form = urlencode(fields) for expected_status in (200, 410): - connection = http.client.HTTPConnection('wordpress', timeout=10) + connection = http.client.HTTPConnection(FIXTURE_HOST, FIXTURE_PORT, timeout=10) connection.request('POST', '/wp-admin/admin-post.php', form, {'Content-Type': 'application/x-www-form-urlencoded'}) response = connection.getresponse() @@ -123,6 +129,26 @@ def concurrent_claim(_): assert claim and claim.target == 'https://http-fixture.example.com/' assert api.claim() is None, 'Concurrent worker obtained a second lease' assert api.start(claim) + +# Per-request admission over the real signed HTTP boundary. +assert api.admit(claim, 'probe') == {'ok': True, 'live': True} +first = parse_grant(api.admit(claim, 'http')) +assert (first.sequence, first.valid_seconds, first.timeout_seconds, first.remaining) == (1, 1.0, 10.0, 19) +try: + api.admit(claim, 'http') +except APIError as error: + assert error.status == 429 and error.payload['reason'] == 'rate_wait', error.payload +else: + raise AssertionError('A second admission inside the pacing interval was granted') +second = AdmissionGate(api, claim).admit() # waits out the interval, then is granted +assert second.sequence == 2 and second.remaining == 18 +bad = dict(id=claim.job_id, lease='f' * 64, scope_hash=claim.scope_hash, kind='http') +try: + api.post('/ct-scan/v1/admit', bad) +except APIError as error: + assert error.status == 410 and error.payload['reason'] == 'lease_invalid_or_expired', error.payload +else: + raise AssertionError('Admission with a foreign lease was granted') checker = Checker(delay=0) checker.opener = SyntheticOpener() result = dataclasses.asdict(checker.check(claim.target)) @@ -134,4 +160,17 @@ def concurrent_claim(_): assert api.result(body) == {'ok': True} assert api.result(body) == {'ok': True}, 'Identical receipt retry must be idempotent' assert api.claim() is None -print('HTTP claim/start/result/receipt-retry passed; scanner used synthetic responses only.') +for kind in ('http', 'probe'): + try: + api.admit(claim, kind) + except APIError as error: + assert error.status == 410 and error.payload['reason'] == 'job_not_running', error.payload + else: + raise AssertionError('A finished job was admitted') +try: + AdmissionGate(api, claim).admit() +except AdmissionDenied as denied: + assert denied.slug == 'job_not_running' +else: + raise AssertionError('The gate admitted a finished job') +print('HTTP claim/start/admit/result/receipt-retry passed; scanner used synthetic responses only.') diff --git a/docs/WORDPRESS.md b/docs/WORDPRESS.md index 291ac6d..4b15802 100644 --- a/docs/WORDPRESS.md +++ b/docs/WORDPRESS.md @@ -32,11 +32,12 @@ flowchart LR confirm. Mailbox control is not independent proof of domain ownership. - The confirmation email also carries a separate Stop link, also hashed at rest. GET only shows a Stop form; a nonce-protected POST cancels the job - (`customer_stopped`) under the queue lock. Claims and starts then refuse it - and late results are rejected. A check the worker already started may finish - its current requests: there is not yet a per-request admission guard. Stopping - a finished job changes nothing and does not block its report. Schema 2 adds - `stop_token_hash`; the upgrade runs automatically on `init`. + (`customer_stopped`) under the queue lock. Claims, starts, results and + request admissions then refuse it (see "Per-request admission"). A request + already admitted when the Stop is recorded may finish within its ten-second + timeout; no further request is admitted. Stopping a finished job changes + nothing and does not block its report. Schema 2 added `stop_token_hash`; + upgrades run automatically on `init`. - Jobs have atomic claims. A claim that was never started can be recovered; a started job with an uncertain outcome is not automatically scanned again. A lost result acknowledgement retries the stored receipt, not the scan. @@ -57,7 +58,7 @@ flowchart LR ## Worker protocol -The three POST routes are `claim`, `start` and `result`, under +The four POST routes are `claim`, `start`, `admit` and `result`, under `/wp-json/ct-scan/v1/`. Each request carries `X-CT-Timestamp` (Unix seconds), `X-CT-Nonce` (32 lowercase hex characters), and `X-CT-Signature` (HMAC-SHA256 hex). The signed string is the @@ -75,6 +76,68 @@ and `scope_hash`. `result` accepts those plus `outcome`, `result`, `error` and The scan output is bounded to 32 KiB. A replayed identical receipt is accepted idempotently; changing its contents is a conflict. +## Per-request admission + +The scanner subprocess holds no key. Before every connection to the customer's +site it asks the root supervisor over a private socket pair; the supervisor +calls the signed `admit` route; WordPress decides under the same queue lock +that serializes Stop, operator revocation, expiry, claims, starts and results. +`admit` accepts `id`, `lease`, `scope_hash` and `kind` (`http` or `probe`). + +`http` is granted only if all of these hold at that moment: the job is +`running` with the presented lease and scope hash, the lease has not lapsed, +the recipient was confirmed, the saved authorization still reads back +unchanged and unexpired (including operator revocation), the job's frozen +profile is known, the previous grant is at least one second old, and fewer than +the profile's budget of requests (20 for `r2-header-v1`: robots.txt up to six +hops plus the homepage up to six hops with one HTTP-to-HTTPS change) were +reserved. The reservation and its time are written in the same compare-and-set +update as the decision, so a lost worker lease, a duplicate delivery or a +restart cannot create a fresh budget. A grant is consumed when issued, is +valid for one second and covers one connection and one request. `probe` only +reports whether the job is still live; it takes no budget and has no pacing. +Refusals use `410` (permanent: cancelled, expired, revoked, lease, profile, +unconfirmed, clock anomaly), `409` (`budget_exhausted`), `429` (`rate_wait` +with `retry_after_ms`, bounded retries) and `503` (`queue_busy`, the lock is +held elsewhere). A `503` for a disabled integration or a schema mismatch also +refuses. Nothing is admitted unless WordPress says so. + +Enforcement in the supervisor and scanner: + +- No grant, no connection: the network adapter asks for admission after + destination validation and before opening any socket, re-checks that the + grant is under one second old immediately before connecting, and reports + completion. A scanner started without the supervisor socket refuses to run. +- One grant is outstanding at a time. A second request, an unexpected message + or an oversized message ends the scan (`admission_protocol_violation`). +- The supervisor measures the disclosed ten-second request bound from the + grant (plus one second of grace) and kills the scanner on overrun + (`request_deadline_exceeded`). +- While the scanner runs, the supervisor probes the job about every two + seconds, including during backoff, and kills the scanner as soon as + WordPress reports that it is cancelled, expired or revoked. The abort is + therefore bounded by the probe interval plus the five-second probe timeout, + and by the ten-second request bound in any case. +- An unreachable guard refuses the next admission (`admission_guard_unavailable`) + and does not extend the bound on a request already admitted. Expired + authorization is refused locally without a call. +- `SubprocessRunner.run` refuses to start without a gate. + +A refusal fails the run with an `admission_` category (for example +`admission_job_cancelled`, `admission_budget_exhausted`). The result for a job +already cancelled is rejected by WordPress as before. A budget or time stop is +a failed receipt, never a clean result, and no automatic rescan follows. + +What this does not do: it cannot constrain a compromised scanner that ignores +the protocol and opens its own sockets. The unprivileged UID and the +memory/process limits help, but egress filtering at the host or container +network layer (allow only the customer's HTTP/80 and HTTPS/443 destinations, +deny private, link-local and metadata ranges) is the necessary second boundary. +`deployment/compose.yaml` does not itself enforce it; that is a deployment +responsibility. DNS lookups of the already authorized hostname happen before +admission and are not treated as target requests. The supervisor therefore also +enforces the deadline and the stop by killing the scanner's process group. + Receipt retries stop at authorization expiry or one hour after lease expiry, whichever comes first. Permanent rejection keeps the receipt in the private quarantine for operator reconciliation; it never initiates a new scan. @@ -212,6 +275,10 @@ The default configuration is disabled, and no historical request sweep exists. runner must authenticate successfully before accepting live submissions. A matching revision and key are required; mismatches fail closed. +Schema 3 adds `profile_id`, `http_admitted` and `last_admit_ms` to the jobs +table (existing rows default to `r2-header-v1` with empty counters); the +upgrade runs on `init` and the worker API refuses requests until it completes. + The healthcheck requires a successful authenticated exchange in the last ten minutes. Use `docker compose -p ct-php2shell -f deployment/compose.yaml ps` and bounded `logs --tail 50` for runner health, and the Promo Requests scan panel diff --git a/integrations/wordpress/ct-scan.php b/integrations/wordpress/ct-scan.php index 9c31e4f..8c61875 100644 --- a/integrations/wordpress/ct-scan.php +++ b/integrations/wordpress/ct-scan.php @@ -17,7 +17,34 @@ } if ( ! defined( 'CT_SCAN_SCHEMA_VERSION' ) ) { - define( 'CT_SCAN_SCHEMA_VERSION', '2' ); + define( 'CT_SCAN_SCHEMA_VERSION', '3' ); +} + +/** + * Execution profiles. A job freezes its profile id at creation, and the limits + * live here in code rather than in editable configuration, so a later setting + * cannot broaden an existing authorization. `r2-header-v1` is the current + * homepage-header screening: robots.txt (at most six hops) plus the homepage + * (at most six hops, with at most one HTTP-to-HTTPS origin change) is at most + * 20 requests. Unknown profiles are refused. + */ +function ct_scan_current_profile() { + return 'r2-header-v1'; +} + +function ct_scan_profile_limits( $profile_id ) { + $profiles = array( + 'r2-header-v1' => array( + 'http_requests' => 20, + // At most one request per second for the target, enforced centrally. + 'min_interval_ms' => 1000, + // Disclosed bounds: an admission may be used for one second, and one + // admitted request may finish within the ten-second timeout. + 'valid_ms' => 1000, + 'timeout_ms' => 10000, + ), + ); + return is_string( $profile_id ) && isset( $profiles[ $profile_id ] ) ? $profiles[ $profile_id ] : false; } /** Load the secret-bearing settings without ever rendering their values. */ @@ -128,6 +155,9 @@ function ct_scan_maybe_install_schema() { report_notification_at datetime NULL, report_delivered_at datetime NULL, revoked_at datetime NULL, + profile_id varchar(32) NOT NULL DEFAULT 'r2-header-v1', + http_admitted smallint(5) unsigned NOT NULL DEFAULT 0, + last_admit_ms bigint(20) unsigned NULL, PRIMARY KEY (job_id), UNIQUE KEY request_id (request_id), KEY status_created (status,created_at), @@ -145,7 +175,7 @@ function ct_scan_maybe_install_schema() { ) {$charset};"; dbDelta( $sql ); $required = array( - $jobs => array( 'job_id', 'request_id', 'target_origin', 'scope_hash', 'terms_version', 'terms_hash', 'terms_accepted_at', 'authorization_expires_at', 'status', 'lease_hash', 'result_json', 'receipt_hash', 'report_delivered_at', 'stop_token_hash' ), + $jobs => array( 'job_id', 'request_id', 'target_origin', 'scope_hash', 'terms_version', 'terms_hash', 'terms_accepted_at', 'authorization_expires_at', 'status', 'lease_hash', 'result_json', 'receipt_hash', 'report_delivered_at', 'stop_token_hash', 'profile_id', 'http_admitted', 'last_admit_ms' ), $nonces => array( 'nonce_hash', 'expires_at' ), ); foreach ( $required as $table => $columns ) { @@ -344,10 +374,14 @@ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_ } /** Acquire the per-site MySQL advisory lock used to serialize state changes. */ -function ct_scan_lock_acquire() { +function ct_scan_lock_acquire( $wait_seconds = 0 ) { global $wpdb; $name = 'ctscan_' . substr( hash( 'sha256', DB_NAME . '|' . $wpdb->prefix ), 0, 24 ); - return '1' === (string) $wpdb->get_var( $wpdb->prepare( 'SELECT GET_LOCK(%s, 0)', $name ) ); + // Worker paths never wait (they fail closed and retry). Customer stop and + // operator revocation may wait briefly so a concurrent admission cannot make + // a cancellation spuriously fail. + $wait = max( 0, min( 5, (int) $wait_seconds ) ); + return '1' === (string) $wpdb->get_var( $wpdb->prepare( 'SELECT GET_LOCK(%s, %d)', $name, $wait ) ); } function ct_scan_lock_release() { @@ -439,8 +473,9 @@ function ct_scan_on_request_ready( $request_id, $fields ) { 'confirmation_token_hash' => hash( 'sha256', $token ), 'confirmation_expires_at' => $confirm_expires, 'stop_token_hash' => hash( 'sha256', $stop_token ), + 'profile_id' => ct_scan_current_profile(), ), - array( '%d', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' ) + array( '%d', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' ) ); if ( 1 !== $inserted ) { $existing = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM {$jobs} WHERE request_id = %d LIMIT 1", $request_id ) ); @@ -711,13 +746,15 @@ function ct_scan_row_by_stop_hash( $hash ) { } /** - * Customer stop: request-bound and idempotent, serialized with claims, starts and results - * by the queue lock. A cancelled job is rejected by the start and result endpoints. A check - * the worker already started may finish its current requests; no new screening starts. + * Customer stop: request-bound and idempotent, serialized with claims, starts, results and + * request admissions by the queue lock. A cancelled job is rejected by the start, admit and + * result endpoints. A request already admitted when the stop commits may finish within its + * ten-second timeout; no further request is admitted. Waits briefly for the lock so that a + * concurrent admission cannot make the customer's stop fail. * Returns 'stopped', 'already_stopped', 'already_finished', or a WP_Error. */ function ct_scan_customer_stop( $job_id, $token ) { - if ( ! ct_scan_lock_acquire() ) { + if ( ! ct_scan_lock_acquire( 3 ) ) { return new WP_Error( 'queue_busy', 'The request is temporarily busy. Please try again.' ); } global $wpdb; @@ -975,6 +1012,119 @@ function ct_scan_start( WP_REST_Request $request ) { } } +/** + * Per-request admission decision. The sandboxed scanner holds no key: before every + * connection to the target, the supervisor asks this guard through the signed + * `admit` route. The caller MUST hold the queue lock, so this decision is + * serialized with customer stop, operator revocation, expiry, lease changes and + * every other admission. Returns array( http_status, payload ). + * + * `http` reserves one request from the frozen profile's budget and records the + * admission time in the same compare-and-set update; the reservation is consumed + * at once and is valid for ct_scan_profile_limits()['valid_ms']. `probe` only + * reports whether the job is still live (no budget, no pacing). + * + * Nothing here is permissive on failure: an unknown profile, a clock that moved + * backwards, a failed write, or an unconfirmed recipient is a refusal. + */ +function ct_scan_admit_locked( $data, $now_ms ) { + global $wpdb; + $table = ct_scan_jobs_table(); + $refuse = static function ( $status, $reason, $extra = array() ) { + return array( (int) $status, array_merge( array( 'ok' => false, 'reason' => sanitize_key( $reason ) ), $extra ) ); + }; + ct_scan_maintenance_locked(); + $row = ct_scan_row( $data['id'] ); + if ( ! $row ) { + return $refuse( 410, 'job_not_found' ); + } + if ( 'cancelled' === $row->status ) { + return $refuse( 410, 'job_cancelled' ); + } + if ( 'expired' === $row->status ) { + return $refuse( 410, 'job_expired' ); + } + if ( 'running' !== $row->status || 1 !== (int) $row->attempts ) { + return $refuse( 410, 'job_not_running' ); + } + if ( empty( $row->lease_hash ) || ! hash_equals( $row->lease_hash, hash( 'sha256', $data['lease'] ) ) + || ! hash_equals( $row->scope_hash, $data['scope_hash'] ) || strtotime( $row->lease_until . ' UTC' ) <= time() ) { + return $refuse( 410, 'lease_invalid_or_expired' ); + } + if ( empty( $row->confirmed_at ) ) { + return $refuse( 410, 'recipient_unconfirmed' ); + } + if ( ! ct_scan_authorization_is_live( $row ) ) { + $wpdb->update( $table, array( 'status' => 'cancelled', 'error_code' => 'authorization_changed', 'finished_at' => ct_scan_utc_now(), 'lease_hash' => null, 'lease_until' => null ), array( 'job_id' => (int) $row->job_id ), array( '%s', '%s', '%s', '%s', '%s' ), array( '%d' ) ); + ct_scan_mark_request_status( (int) $row->request_id, 'cancelled', 'authorization_changed' ); + return $refuse( 410, 'authorization_revoked_or_expired' ); + } + $limits = ct_scan_profile_limits( $row->profile_id ); + if ( ! $limits ) { + return $refuse( 410, 'profile_unknown' ); + } + if ( 'probe' === $data['kind'] ) { + return array( 200, array( 'ok' => true, 'live' => true ) ); + } + $last = null === $row->last_admit_ms ? null : (int) $row->last_admit_ms; + if ( null !== $last ) { + if ( $now_ms + $limits['min_interval_ms'] < $last ) { + // The clock moved backwards by more than the pacing interval: do not guess. + return $refuse( 410, 'clock_anomaly' ); + } + $wait = $last + $limits['min_interval_ms'] - $now_ms; + if ( $wait > 0 ) { + return $refuse( 429, 'rate_wait', array( 'retry_after_ms' => (int) min( $wait, $limits['min_interval_ms'] ) ) ); + } + } + $used = (int) $row->http_admitted; + if ( $used >= $limits['http_requests'] ) { + return $refuse( 409, 'budget_exhausted' ); + } + $updated = $wpdb->update( + $table, + array( 'http_admitted' => $used + 1, 'last_admit_ms' => $now_ms ), + array( 'job_id' => (int) $row->job_id, 'status' => 'running', 'lease_hash' => $row->lease_hash, 'http_admitted' => $used ), + array( '%d', '%d' ), + array( '%d', '%s', '%s', '%d' ) + ); + if ( 1 !== $updated ) { + return $refuse( 503, 'admission_write_failed' ); + } + return array( 200, array( + 'ok' => true, + 'admitted' => true, + 'sequence' => $used + 1, + 'valid_ms' => (int) $limits['valid_ms'], + 'timeout_ms' => (int) $limits['timeout_ms'], + 'http_remaining' => (int) ( $limits['http_requests'] - $used - 1 ), + ) ); +} + +function ct_scan_admit( WP_REST_Request $request ) { + $body = ct_scan_rest_authenticate( $request, '/ct-scan/v1/admit' ); + if ( is_wp_error( $body ) ) { + return ct_scan_rest_error( (int) $body->get_error_data()['status'], $body->get_error_code() ); + } + $data = ct_scan_decode_request_body( $body ); + if ( is_wp_error( $data ) || ! ct_scan_expected_keys( $data, array( 'id', 'lease', 'scope_hash', 'kind' ) ) || ! ct_scan_callback_payload( $data ) + || ! is_string( $data['kind'] ) || ! in_array( $data['kind'], array( 'http', 'probe' ), true ) ) { + return ct_scan_rest_error( 400, 'request_invalid' ); + } + if ( ! ct_scan_lock_acquire() ) { + // Fail closed: a refusal here ends the screening rather than risking an unserialized send. + return ct_scan_rest_response( 503, array( 'ok' => false, 'reason' => 'queue_busy' ) ); + } + try { + $decision = ct_scan_admit_locked( $data, (int) floor( microtime( true ) * 1000 ) ); + } catch ( Throwable $error ) { + $decision = array( 503, array( 'ok' => false, 'reason' => 'admission_error' ) ); + } finally { + ct_scan_lock_release(); + } + return ct_scan_rest_response( $decision[0], $decision[1] ); +} + function ct_scan_clean_result_line( $value, $limit = 500 ) { return is_string( $value ) && strlen( $value ) <= $limit; } @@ -1181,6 +1331,11 @@ function ct_scan_register_rest_routes() { 'callback' => 'ct_scan_start', 'permission_callback' => '__return_true', ) ); + register_rest_route( 'ct-scan/v1', '/admit', array( + 'methods' => WP_REST_Server::CREATABLE, + 'callback' => 'ct_scan_admit', + 'permission_callback' => '__return_true', + ) ); register_rest_route( 'ct-scan/v1', '/result', array( 'methods' => WP_REST_Server::CREATABLE, 'callback' => 'ct_scan_result', @@ -1275,7 +1430,7 @@ function ct_scan_build_report_body( $row ) { } function ct_scan_admin_revoke( $job_id ) { - if ( ! ct_scan_lock_acquire() ) { + if ( ! ct_scan_lock_acquire( 3 ) ) { return new WP_Error( 'queue_busy', 'The queue is temporarily busy.' ); } global $wpdb; diff --git a/integrations/wordpress/tests/admission-disabled.php b/integrations/wordpress/tests/admission-disabled.php new file mode 100644 index 0000000..ef9e606 --- /dev/null +++ b/integrations/wordpress/tests/admission-disabled.php @@ -0,0 +1,7 @@ + 1, 'lease' => str_repeat( 'a', 64 ), 'scope_hash' => str_repeat( 'b', 64 ), 'kind' => 'http' ) ); +ct_test_assert( 503 === $response->get_status() && 'ct_scan_disabled' === ( $response->get_data()['error'] ?? '' ), 'A disabled integration admitted or mis-reported a request.' ); +echo "disabled\n"; diff --git a/integrations/wordpress/tests/admission.php b/integrations/wordpress/tests/admission.php new file mode 100644 index 0000000..d6e3098 --- /dev/null +++ b/integrations/wordpress/tests/admission.php @@ -0,0 +1,220 @@ +update( ct_scan_jobs_table(), array( 'status' => 'failed', 'outcome' => 'failed', 'finished_at' => ct_scan_utc_now(), 'lease_hash' => null, 'lease_until' => null ), array( 'job_id' => (int) $job_id ) ); +} + +function ct_test_stop_token( $job_id ) { + // The raw token exists only in the customer's email; a test token is installed by hash. + global $wpdb; + $token = bin2hex( random_bytes( 32 ) ); + $wpdb->update( ct_scan_jobs_table(), array( 'stop_token_hash' => hash( 'sha256', $token ) ), array( 'job_id' => (int) $job_id ) ); + return $token; +} + +// Schema 3 columns exist and the schema option is current. +$columns = $wpdb->get_col( 'SHOW COLUMNS FROM ' . $jobs, 0 ); +foreach ( array( 'profile_id', 'http_admitted', 'last_admit_ms' ) as $column ) { + ct_test_assert( in_array( $column, $columns, true ), "Schema is missing {$column}." ); +} +ct_test_assert( get_option( 'ct_scan_schema_version' ) === CT_SCAN_SCHEMA_VERSION, 'Schema version option is not current.' ); + +// Authentication and shape: unsigned, replayed, extra-key and unknown-kind requests never admit. +$bad = new WP_REST_Request( 'POST', '/ct-scan/v1/admit' ); +$bad->set_body( '{}' ); +ct_test_assert( 401 === ct_scan_admit( $bad )->get_status(), 'An unsigned admission request was not rejected.' ); + +$job1 = ct_test_running_job( 'admit-one.example.net' ); +$request_ids[] = $job1['request_id']; +$row = ct_test_job_fields( $job1['job_id'] ); +ct_test_assert( 'r2-header-v1' === $row->profile_id && 0 === (int) $row->http_admitted && null === $row->last_admit_ms, 'New job must freeze its profile with empty counters.' ); +ct_test_assert( 400 === ct_test_admit( $job1, 'http', array( 'extra' => 1 ) )->get_status(), 'Extra fields must be rejected.' ); +ct_test_assert( 400 === ct_test_admit( $job1, 'tls' )->get_status(), 'An unknown admission kind must be rejected.' ); +ct_test_assert( 400 === ct_test_admit( $job1, 'http', array( 'kind' => array( 'http' ) ) )->get_status(), 'A non-string kind must be rejected.' ); +$replay_nonce = bin2hex( random_bytes( 16 ) ); +$payload = array( 'id' => $job1['job']['id'], 'lease' => $job1['job']['lease'], 'scope_hash' => $job1['job']['scope_hash'], 'kind' => 'probe' ); +ct_test_assert( 200 === ct_test_rest_call( 'ct_scan_admit', '/ct-scan/v1/admit', $payload, $replay_nonce )->get_status(), 'A signed probe was refused.' ); +ct_test_assert( 409 === ct_test_rest_call( 'ct_scan_admit', '/ct-scan/v1/admit', $payload, $replay_nonce )->get_status(), 'A replayed admission request was accepted.' ); +ct_test_assert( 401 === ct_test_rest_call( 'ct_scan_admit', '/ct-scan/v1/admit', $payload, null, time() - 4000 )->get_status(), 'A stale-timestamp admission request was accepted.' ); + +// Wrong lease or scope: refused, no counter change. +$wrong = ct_test_admit( $job1, 'http', array( 'lease' => str_repeat( 'c', 64 ) ) ); +ct_test_assert( 410 === $wrong->get_status() && 'lease_invalid_or_expired' === $wrong->get_data()['reason'], 'A wrong lease was admitted.' ); +$wrong = ct_test_admit( $job1, 'http', array( 'scope_hash' => str_repeat( 'd', 64 ) ) ); +ct_test_assert( 410 === $wrong->get_status(), 'A wrong scope hash was admitted.' ); +ct_test_assert( 0 === (int) ct_test_job_fields( $job1['job_id'] )->http_admitted, 'A refused admission consumed budget.' ); + +// First admission is granted and recorded before it is returned; probes do not consume or pace. +$grant = ct_test_admit( $job1 ); +$data = $grant->get_data(); +ct_test_assert( 200 === $grant->get_status() && true === $data['ok'] && true === $data['admitted'] && 1 === $data['sequence'] && 1000 === $data['valid_ms'] && 10000 === $data['timeout_ms'] && 19 === $data['http_remaining'], 'First admission was not granted with the disclosed bounds.' ); +$row = ct_test_job_fields( $job1['job_id'] ); +ct_test_assert( 1 === (int) $row->http_admitted && null !== $row->last_admit_ms, 'The reservation was not persisted with the grant.' ); +$probe = ct_test_admit( $job1, 'probe' ); +ct_test_assert( 200 === $probe->get_status() && true === $probe->get_data()['live'] && 1 === (int) ct_test_job_fields( $job1['job_id'] )->http_admitted, 'A probe consumed budget or reported a live job incorrectly.' ); + +// Pacing: a second request inside the interval is told to wait and consumes nothing. +$early = ct_test_admit( $job1 ); +ct_test_assert( 429 === $early->get_status() && 'rate_wait' === $early->get_data()['reason'] && $early->get_data()['retry_after_ms'] > 0 && $early->get_data()['retry_after_ms'] <= 1000, 'Pacing did not require a wait.' ); +ct_test_assert( 1 === (int) ct_test_job_fields( $job1['job_id'] )->http_admitted, 'A paced request consumed budget.' ); +ct_test_age_admission( $job1['job_id'] ); +$second = ct_test_admit( $job1 ); +ct_test_assert( 200 === $second->get_status() && 2 === $second->get_data()['sequence'], 'Admission after the interval was refused.' ); + +// A clock that moved backwards by more than the interval refuses rather than guessing. +$wpdb->query( $wpdb->prepare( "UPDATE {$jobs} SET last_admit_ms = last_admit_ms + 3600000 WHERE job_id = %d", $job1['job_id'] ) ); +$anomaly = ct_test_admit( $job1 ); +ct_test_assert( 410 === $anomaly->get_status() && 'clock_anomaly' === $anomaly->get_data()['reason'], 'A backwards clock was not refused.' ); +$wpdb->query( $wpdb->prepare( "UPDATE {$jobs} SET last_admit_ms = last_admit_ms - 3600000 WHERE job_id = %d", $job1['job_id'] ) ); + +// Budget: exhausted at the frozen limit; refusal is stable and does not reset. +ct_test_age_admission( $job1['job_id'] ); +$wpdb->update( $jobs, array( 'http_admitted' => 19 ), array( 'job_id' => $job1['job_id'] ) ); +$last = ct_test_admit( $job1 ); +ct_test_assert( 200 === $last->get_status() && 0 === $last->get_data()['http_remaining'], 'The final budgeted request was refused.' ); +ct_test_age_admission( $job1['job_id'] ); +$over = ct_test_admit( $job1 ); +ct_test_assert( 409 === $over->get_status() && 'budget_exhausted' === $over->get_data()['reason'] && 20 === (int) ct_test_job_fields( $job1['job_id'] )->http_admitted, 'The request budget was exceeded.' ); + +// Duplicate delivery of the ready event neither creates a job nor resets counters, profile or expiry. +delete_post_meta( $job1['request_id'], '_ct_scan_event_recorded' ); +$expiry_before = ct_test_job_fields( $job1['job_id'] )->authorization_expires_at; +$dup = ct_scan_on_request_ready( $job1['request_id'], $job1['fields'] ); +$after = ct_test_job_fields( $job1['job_id'] ); +ct_test_assert( ! empty( $dup['duplicate'] ) && 20 === (int) $after->http_admitted && $expiry_before === $after->authorization_expires_at && 'running' === $after->status, 'A duplicate event reset the job budget, expiry or state.' ); + +// An unknown profile is refused even though the job is otherwise live. +$wpdb->update( $jobs, array( 'profile_id' => 'unknown-v9', 'http_admitted' => 0 ), array( 'job_id' => $job1['job_id'] ) ); +ct_test_age_admission( $job1['job_id'] ); +$unknown = ct_test_admit( $job1 ); +ct_test_assert( 410 === $unknown->get_status() && 'profile_unknown' === $unknown->get_data()['reason'], 'An unknown profile was admitted.' ); +$wpdb->update( $jobs, array( 'profile_id' => 'r2-header-v1' ), array( 'job_id' => $job1['job_id'] ) ); + +// An unconfirmed recipient is never admitted. +$wpdb->update( $jobs, array( 'confirmed_at' => null ), array( 'job_id' => $job1['job_id'] ) ); +$unconfirmed = ct_test_admit( $job1 ); +ct_test_assert( 410 === $unconfirmed->get_status() && 'recipient_unconfirmed' === $unconfirmed->get_data()['reason'], 'An unconfirmed job was admitted.' ); +$wpdb->update( $jobs, array( 'confirmed_at' => ct_scan_utc_now() ), array( 'job_id' => $job1['job_id'] ) ); + +// Saved authorization changed after the job began: refused and the job is cancelled. +$original_hash = get_post_meta( $job1['request_id'], '_ct_terms_sha256', true ); +update_post_meta( $job1['request_id'], '_ct_terms_sha256', str_repeat( '9', 64 ) ); +$changed = ct_test_admit( $job1 ); +ct_test_assert( 410 === $changed->get_status() && 'authorization_revoked_or_expired' === $changed->get_data()['reason'] && 'cancelled' === ct_test_job_fields( $job1['job_id'] )->status, 'A changed authorization was admitted.' ); +update_post_meta( $job1['request_id'], '_ct_terms_sha256', $original_hash ); +$after_change = ct_test_admit( $job1 ); +ct_test_assert( 410 === $after_change->get_status(), 'A cancelled job became admissible again.' ); + +// Before start: a claimed (leased) job and a queued job have no admission. +list( $leased_request, $leased_fields ) = ct_test_request( 'admit-leased.example.net' ); +$request_ids[] = $leased_request; +$leased_event = ct_scan_on_request_ready( $leased_request, $leased_fields ); +ct_test_queue_job( $leased_event['job_id'] ); +$claimed = ct_test_rest_call( 'ct_scan_claim', '/ct-scan/v1/claim', array() )->get_data()['job']; +$leased = array( 'job' => $claimed, 'job_id' => (int) $leased_event['job_id'] ); +$pre = ct_test_admit( $leased ); +ct_test_assert( 410 === $pre->get_status() && 'job_not_running' === $pre->get_data()['reason'] && 0 === (int) ct_test_job_fields( $leased['job_id'] )->http_admitted, 'A leased-but-unstarted job was admitted.' ); +$stop_token = ct_test_stop_token( $leased['job_id'] ); +ct_test_assert( 'stopped' === ct_scan_customer_stop( $leased['job_id'], $stop_token ), 'Fixture could not stop the leased job.' ); +ct_test_assert( 410 === ct_test_admit( $leased )->get_status(), 'A stopped, unstarted job was admitted.' ); + +// Cancellation/admission race, order 1: stop commits first, so the next admission is refused. +$job2 = ct_test_running_job( 'admit-stop-first.example.net' ); +$request_ids[] = $job2['request_id']; +$stop2 = ct_test_stop_token( $job2['job_id'] ); +ct_test_assert( 'stopped' === ct_scan_customer_stop( $job2['job_id'], $stop2 ), 'Customer stop failed on a running job.' ); +$refused = ct_test_admit( $job2 ); +ct_test_assert( 410 === $refused->get_status() && 'job_cancelled' === $refused->get_data()['reason'] && 0 === (int) ct_test_job_fields( $job2['job_id'] )->http_admitted, 'An admission was granted after a recorded customer stop.' ); +ct_test_assert( 410 === ct_test_admit( $job2, 'probe' )->get_status(), 'A probe reported a stopped job as live.' ); + +// Order 2: the admission commits first, so exactly that one request may finish; every later one is refused. +$job3 = ct_test_running_job( 'admit-grant-first.example.net' ); +$request_ids[] = $job3['request_id']; +$stop3 = ct_test_stop_token( $job3['job_id'] ); +ct_test_assert( 200 === ct_test_admit( $job3 )->get_status(), 'Fixture admission before stop was refused.' ); +ct_test_assert( 'stopped' === ct_scan_customer_stop( $job3['job_id'], $stop3 ), 'Customer stop after an admission failed.' ); +ct_test_age_admission( $job3['job_id'] ); +ct_test_assert( 410 === ct_test_admit( $job3 )->get_status() && 410 === ct_test_admit( $job3, 'probe' )->get_status(), 'A request was admitted after the stop that followed an admission.' ); +ct_test_assert( 1 === (int) ct_test_job_fields( $job3['job_id'] )->http_admitted, 'Exactly the one already-admitted request should be counted.' ); + +// Order 3, real contention: a second process holds the queue lock (an admission in progress). +// An admission cannot proceed unserialized (busy, nothing written); the stop waits for the lock, +// wins, and the next admission is refused. +$job4 = ct_test_running_job( 'admit-contended.example.net' ); +$request_ids[] = $job4['request_id']; +$stop4 = ct_test_stop_token( $job4['job_id'] ); +$holder = proc_open( array( PHP_BINARY, __DIR__ . '/lock-holder.php', '1.3' ), array( 1 => array( 'pipe', 'w' ), 2 => array( 'pipe', 'w' ) ), $pipes, null, null ); +ct_test_assert( is_resource( $holder ) && "locked\n" === fgets( $pipes[1] ), 'The lock-holder helper did not acquire the queue lock.' ); +$busy = ct_test_admit( $job4 ); +ct_test_assert( 503 === $busy->get_status() && 'queue_busy' === $busy->get_data()['reason'] && 0 === (int) ct_test_job_fields( $job4['job_id'] )->http_admitted, 'An admission proceeded while the queue lock was held elsewhere.' ); +$started = microtime( true ); +$stopped = ct_scan_customer_stop( $job4['job_id'], $stop4 ); +$elapsed = microtime( true ) - $started; +ct_test_assert( 'stopped' === $stopped && $elapsed > 0.3, 'The stop did not wait for the in-progress lock holder and then succeed.' ); +ct_test_assert( 410 === ct_test_admit( $job4 )->get_status(), 'An admission followed a stop that waited on the lock.' ); +fclose( $pipes[1] ); +fclose( $pipes[2] ); +proc_close( $holder ); + +// Operator revocation behaves like a stop for admission. +$job5 = ct_test_running_job( 'admit-revoked.example.net' ); +$request_ids[] = $job5['request_id']; +ct_test_assert( true === ct_scan_admin_revoke( $job5['job_id'] ), 'Operator revocation failed.' ); +ct_test_assert( 410 === ct_test_admit( $job5 )->get_status(), 'A revoked job was admitted.' ); + +// Expiry: authorization expiry blocks the next admission; lease expiry cannot start a fresh budget. +$job6 = ct_test_running_job( 'admit-expiry.example.net' ); +$request_ids[] = $job6['request_id']; +ct_test_assert( 200 === ct_test_admit( $job6 )->get_status(), 'Fixture admission before expiry was refused.' ); +$wpdb->update( $jobs, array( 'authorization_expires_at' => gmdate( 'Y-m-d H:i:s', time() - 1 ) ), array( 'job_id' => $job6['job_id'] ) ); +ct_test_age_admission( $job6['job_id'] ); +$expired = ct_test_admit( $job6 ); +ct_test_assert( 410 === $expired->get_status() && 'job_expired' === $expired->get_data()['reason'], 'An admission was granted after authorization expiry.' ); +$job7 = ct_test_running_job( 'admit-lease.example.net' ); +$request_ids[] = $job7['request_id']; +ct_test_assert( 200 === ct_test_admit( $job7 )->get_status(), 'Fixture admission before lease expiry was refused.' ); +$wpdb->update( $jobs, array( 'lease_until' => gmdate( 'Y-m-d H:i:s', time() - 2 ) ), array( 'job_id' => $job7['job_id'] ) ); +ct_test_age_admission( $job7['job_id'] ); +$lapsed = ct_test_admit( $job7 ); +ct_test_assert( 410 === $lapsed->get_status() && 1 === (int) ct_test_job_fields( $job7['job_id'] )->http_admitted && 'failed_unknown' === ct_test_job_fields( $job7['job_id'] )->status, 'A lapsed lease was admitted or its counters reset.' ); +$reclaim = ct_test_rest_call( 'ct_scan_claim', '/ct-scan/v1/claim', array() ); +ct_test_assert( null === $reclaim->get_data()['job'] && 1 === (int) ct_test_job_fields( $job7['job_id'] )->http_admitted, 'Recovery produced a fresh claim or budget for a started job.' ); + +// Unavailable authorization store: a schema mismatch disables the whole worker API, including admission. +$job8 = ct_test_running_job( 'admit-store.example.net' ); +$request_ids[] = $job8['request_id']; +update_option( 'ct_scan_schema_version', '2', false ); +$unavailable = ct_test_admit( $job8 ); +update_option( 'ct_scan_schema_version', CT_SCAN_SCHEMA_VERSION, false ); +ct_test_assert( 503 === $unavailable->get_status() && 0 === (int) ct_test_job_fields( $job8['job_id'] )->http_admitted, 'Admission was granted while the store was not at the expected schema.' ); +ct_test_end_job( $job8['job_id'] ); + +// Global operator kill switch: with the integration disabled, nothing is admitted. +$disabled = proc_open( array( PHP_BINARY, __DIR__ . '/admission-disabled.php' ), array( 1 => array( 'pipe', 'w' ), 2 => array( 'pipe', 'w' ) ), $dpipes, null, array_merge( $_ENV, array( 'CT_TEST_ENABLED' => '0', 'WORDPRESS_DB_PASSWORD' => 'fixture-only', 'WORDPRESS_DB_HOST' => 'db', 'CT_TEST_WP_ROOT' => getenv( 'CT_TEST_WP_ROOT' ) ?: '/var/www/html', 'PATH' => getenv( 'PATH' ) ) ) ); +$out = stream_get_contents( $dpipes[1] ); +$err = stream_get_contents( $dpipes[2] ); +proc_close( $disabled ); +ct_test_assert( "disabled\n" === $out, 'The disabled-integration check failed: ' . $out . $err ); + +// Remove only records created by this test. +foreach ( $request_ids as $id ) { + $wpdb->delete( $jobs, array( 'request_id' => $id ), array( '%d' ) ); + wp_delete_post( $id, true ); +} +echo "Admission guard checks passed.\n"; diff --git a/integrations/wordpress/tests/integration.php b/integrations/wordpress/tests/integration.php index 49230b7..050d364 100644 --- a/integrations/wordpress/tests/integration.php +++ b/integrations/wordpress/tests/integration.php @@ -3,95 +3,7 @@ * Disposable WordPress + MariaDB integration checks for the MU plugin. * Run only with the repository's `fixture-only` Docker Compose environment. */ -if ( PHP_SAPI !== 'cli' || getenv( 'WORDPRESS_DB_PASSWORD' ) !== 'fixture-only' || getenv( 'WORDPRESS_DB_HOST' ) !== 'db' ) { - throw new RuntimeException( 'Refusing to run outside the disposable WordPress fixture.' ); -} - -$private_config = tempnam( '/tmp', 'ct-scan-integration-' ); -file_put_contents( $private_config, ' true, - 'supported_terms_version' => '2026-10-21-r2', - 'expected_checker_revision' => str_repeat( 'a', 40 ), - 'expected_rules_as_of' => '2026-10-21', - 'worker_hmac_secret' => str_repeat( 'fixture-key-', 4 ), - 'authorization_end_at' => gmdate( 'Y-m-d\TH:i:s\Z', time() + 30 * 86400 ), - 'confirmation_ttl_hours' => 24, - 'global_daily_limit' => 1000, - 'target_daily_limit' => 100, -), true ) . ';' ); -define( 'CT_SCAN_PRIVATE_CONFIG', $private_config ); -register_shutdown_function( static function () use ( $private_config ) { @unlink( $private_config ); } ); - -$_SERVER['HTTP_HOST'] = 'wordpress'; -require '/var/www/html/wp-load.php'; -ct_scan_maybe_install_schema(); - -function ct_test_assert( $condition, $message ) { - if ( ! $condition ) { - throw new RuntimeException( $message ); - } -} - -function ct_test_request( $host, $email = 'fixture@example.com' ) { - $target = 'https://' . $host . '/'; - $contract = "Client contact: Fixture Reviewer\nClient email: {$email}\nWebsite (scan scope): {$target}\nReferred by: direct\nTerms version: 2026-10-21-r2\nAccepted test fixture terms."; - $id = wp_insert_post( array( - 'post_type' => 'ct_promo_request', - 'post_status' => 'private', - 'post_title' => 'Integration fixture request', - 'post_content' => "TERMS AS ACCEPTED\n\n" . $contract, - ), true ); - if ( is_wp_error( $id ) ) { - throw new RuntimeException( $id->get_error_message() ); - } - $meta = array( - '_ct_scan_target' => $target, - '_ct_scan_email' => $email, - '_ct_scan_name' => 'Fixture Reviewer', - '_ct_ref' => 'direct', - '_ct_terms_saved' => '1', - '_ct_terms_version' => '2026-10-21-r2', - '_ct_terms_url' => 'https://complete.tech/php2shell-scan-terms/', - '_ct_terms_sha256' => hash( 'sha256', 'Accepted test fixture terms.' ), - '_ct_terms_accepted_at' => gmdate( 'Y-m-d\TH:i:s\Z', time() - 30 ), - '_ct_terms_contract' => $contract, - ); - foreach ( $meta as $key => $value ) { - update_post_meta( $id, $key, $value ); - } - return array( (int) $id, array( 'target' => $target, 'email' => $email, 'name' => 'Fixture Reviewer', 'referrer' => 'direct' ) ); -} - -function ct_test_rest_call( $callback, $route, $payload, $nonce = null, $timestamp = null ) { - $body = wp_json_encode( $payload, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); - $timestamp = null === $timestamp ? (string) time() : (string) $timestamp; - $nonce = null === $nonce ? bin2hex( random_bytes( 16 ) ) : $nonce; - $signing = $timestamp . "\n" . strtolower( $nonce ) . "\nPOST\n" . $route . "\n" . hash( 'sha256', $body ); - $request = new WP_REST_Request( 'POST', $route ); - $request->set_body( $body ); - $request->set_header( 'X-CT-Timestamp', $timestamp ); - $request->set_header( 'X-CT-Nonce', $nonce ); - $request->set_header( 'X-CT-Signature', hash_hmac( 'sha256', $signing, ct_scan_config()['worker_hmac_secret'] ) ); - return call_user_func( $callback, $request ); -} - -function ct_test_result( $target ) { - return array( - 'input_url' => $target, 'final_url' => $target, 'status' => 'wordpress', 'fetch_status' => 'ok', 'http_status' => 200, - 'wordpress_version' => '6.8', 'php_version' => '8.2.1', 'server' => 'fixture', 'powered_by' => 'PHP/8.2.1', 'title' => 'Fixture', - 'evidence' => array( 'generator present' ), 'asset_version_hints' => array(), 'notes' => array(), 'checked_at' => gmdate( 'Y-m-d\TH:i:s\Z' ), - 'php_version_candidates' => array( '8.2.1' ), 'php_evidence' => array( 'X-Powered-By: PHP/8.2.1 ' ), - 'php2shell_status' => 'potentially_affected', 'php2shell_version_status' => 'below_upstream_fix', 'php2shell_fixed_version' => '8.2.31', - 'php2shell_exploitability' => 'not_tested', 'php2shell_cves' => array( 'CVE-2026-109618', 'CVE-2026-109619' ), - 'php2shell_rules_as_of' => '2026-10-21', 'php2shell_sources' => array( 'https://www.php.net/' ), 'php2shell_notes' => array( 'Public evidence only.' ), - 'php2shell_recommendation' => 'Verify the deployed package and vendor backports.', - ); -} - -function ct_test_queue_job( $job_id ) { - global $wpdb; - $wpdb->update( ct_scan_jobs_table(), array( 'status' => 'queued', 'confirmation_token_hash' => null, 'confirmation_expires_at' => null, 'confirmed_at' => ct_scan_utc_now() ), array( 'job_id' => (int) $job_id ) ); -} +require __DIR__ . '/support.php'; // Canonical scope normalization and control-character rejection. ct_test_assert( 'https://example.com/' === ct_scan_normalize_target( 'https://EXAMPLE.com/' ), 'Canonical origin must have lowercase host and trailing slash.' ); @@ -101,14 +13,7 @@ function ct_test_queue_job( $job_id ) { global $wpdb; $jobs = ct_scan_jobs_table(); -$old_fixture_ids = $wpdb->get_col( $wpdb->prepare( "SELECT request_id FROM {$jobs} WHERE contact_name = %s", 'Fixture Reviewer' ) ); -foreach ( (array) $old_fixture_ids as $old_id ) { - $old_post = get_post( (int) $old_id ); - if ( $old_post && 'ct_promo_request' === $old_post->post_type && 'private' === $old_post->post_status && 'Integration fixture request' === $old_post->post_title ) { - $wpdb->delete( $jobs, array( 'request_id' => (int) $old_id ), array( '%d' ) ); - wp_delete_post( (int) $old_id, true ); - } -} +ct_test_purge_fixtures(); $request_ids = array(); $mail_calls = array(); $mail_mode = true; diff --git a/integrations/wordpress/tests/lock-holder.php b/integrations/wordpress/tests/lock-holder.php new file mode 100644 index 0000000..3ad867b --- /dev/null +++ b/integrations/wordpress/tests/lock-holder.php @@ -0,0 +1,12 @@ + '0' !== getenv( 'CT_TEST_ENABLED' ), + 'supported_terms_version' => '2026-10-21-r2', + 'expected_checker_revision' => str_repeat( 'a', 40 ), + 'expected_rules_as_of' => '2026-10-21', + 'worker_hmac_secret' => str_repeat( 'fixture-key-', 4 ), + 'authorization_end_at' => gmdate( 'Y-m-d\TH:i:s\Z', time() + 30 * 86400 ), + 'confirmation_ttl_hours' => 24, + 'global_daily_limit' => 1000, + 'target_daily_limit' => 100, +), true ) . ';' ); +define( 'CT_SCAN_PRIVATE_CONFIG', $private_config ); +register_shutdown_function( static function () use ( $private_config ) { @unlink( $private_config ); } ); + +$_SERVER['HTTP_HOST'] = 'wordpress'; +require ( getenv( 'CT_TEST_WP_ROOT' ) ?: '/var/www/html' ) . '/wp-load.php'; +ct_scan_maybe_install_schema(); + +function ct_test_assert( $condition, $message ) { + if ( ! $condition ) { + throw new RuntimeException( $message ); + } +} + +function ct_test_request( $host, $email = 'fixture@example.com' ) { + $target = 'https://' . $host . '/'; + $contract = "Client contact: Fixture Reviewer\nClient email: {$email}\nWebsite (scan scope): {$target}\nReferred by: direct\nTerms version: 2026-10-21-r2\nAccepted test fixture terms."; + $id = wp_insert_post( array( + 'post_type' => 'ct_promo_request', + 'post_status' => 'private', + 'post_title' => 'Integration fixture request', + 'post_content' => "TERMS AS ACCEPTED\n\n" . $contract, + ), true ); + if ( is_wp_error( $id ) ) { + throw new RuntimeException( $id->get_error_message() ); + } + $meta = array( + '_ct_scan_target' => $target, + '_ct_scan_email' => $email, + '_ct_scan_name' => 'Fixture Reviewer', + '_ct_ref' => 'direct', + '_ct_terms_saved' => '1', + '_ct_terms_version' => '2026-10-21-r2', + '_ct_terms_url' => 'https://complete.tech/php2shell-scan-terms/', + '_ct_terms_sha256' => hash( 'sha256', 'Accepted test fixture terms.' ), + '_ct_terms_accepted_at' => gmdate( 'Y-m-d\TH:i:s\Z', time() - 30 ), + '_ct_terms_contract' => $contract, + ); + foreach ( $meta as $key => $value ) { + update_post_meta( $id, $key, $value ); + } + return array( (int) $id, array( 'target' => $target, 'email' => $email, 'name' => 'Fixture Reviewer', 'referrer' => 'direct' ) ); +} + +function ct_test_rest_call( $callback, $route, $payload, $nonce = null, $timestamp = null ) { + $body = wp_json_encode( $payload, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); + $timestamp = null === $timestamp ? (string) time() : (string) $timestamp; + $nonce = null === $nonce ? bin2hex( random_bytes( 16 ) ) : $nonce; + $signing = $timestamp . "\n" . strtolower( $nonce ) . "\nPOST\n" . $route . "\n" . hash( 'sha256', $body ); + $request = new WP_REST_Request( 'POST', $route ); + $request->set_body( $body ); + $request->set_header( 'X-CT-Timestamp', $timestamp ); + $request->set_header( 'X-CT-Nonce', $nonce ); + $request->set_header( 'X-CT-Signature', hash_hmac( 'sha256', $signing, ct_scan_config()['worker_hmac_secret'] ) ); + return call_user_func( $callback, $request ); +} + +function ct_test_result( $target ) { + return array( + 'input_url' => $target, 'final_url' => $target, 'status' => 'wordpress', 'fetch_status' => 'ok', 'http_status' => 200, + 'wordpress_version' => '6.8', 'php_version' => '8.2.1', 'server' => 'fixture', 'powered_by' => 'PHP/8.2.1', 'title' => 'Fixture', + 'evidence' => array( 'generator present' ), 'asset_version_hints' => array(), 'notes' => array(), 'checked_at' => gmdate( 'Y-m-d\TH:i:s\Z' ), + 'php_version_candidates' => array( '8.2.1' ), 'php_evidence' => array( 'X-Powered-By: PHP/8.2.1 ' ), + 'php2shell_status' => 'potentially_affected', 'php2shell_version_status' => 'below_upstream_fix', 'php2shell_fixed_version' => '8.2.31', + 'php2shell_exploitability' => 'not_tested', 'php2shell_cves' => array( 'CVE-2026-109618', 'CVE-2026-109619' ), + 'php2shell_rules_as_of' => '2026-10-21', 'php2shell_sources' => array( 'https://www.php.net/' ), 'php2shell_notes' => array( 'Public evidence only.' ), + 'php2shell_recommendation' => 'Verify the deployed package and vendor backports.', + ); +} + +function ct_test_queue_job( $job_id ) { + global $wpdb; + $wpdb->update( ct_scan_jobs_table(), array( 'status' => 'queued', 'confirmation_token_hash' => null, 'confirmation_expires_at' => null, 'confirmed_at' => ct_scan_utc_now() ), array( 'job_id' => (int) $job_id ) ); +} + + +/** Create a fixture request, confirm it, claim and start it. Returns the running job and its worker lease. */ +function ct_test_running_job( $host ) { + global $wpdb; + list( $request_id, $fields ) = ct_test_request( $host ); + $event = ct_scan_on_request_ready( $request_id, $fields ); + ct_test_assert( is_array( $event ) && empty( $event['duplicate'] ), 'Fixture job was not created.' ); + ct_test_queue_job( $event['job_id'] ); + $claim = ct_test_rest_call( 'ct_scan_claim', '/ct-scan/v1/claim', array() ); + $job = $claim->get_data()['job']; + ct_test_assert( is_array( $job ) && (int) $job['id'] === (int) $event['job_id'], 'Fixture expected to claim its own job.' ); + $start = ct_test_rest_call( 'ct_scan_start', '/ct-scan/v1/start', array( 'id' => $job['id'], 'lease' => $job['lease'], 'scope_hash' => $job['scope_hash'] ) ); + ct_test_assert( 200 === $start->get_status(), 'Fixture job did not start.' ); + return array( 'request_id' => $request_id, 'fields' => $fields, 'job_id' => (int) $event['job_id'], 'job' => $job ); +} + +function ct_test_admit( $running, $kind = 'http', $override = array() ) { + $payload = array_merge( array( 'id' => $running['job']['id'], 'lease' => $running['job']['lease'], 'scope_hash' => $running['job']['scope_hash'], 'kind' => $kind ), $override ); + return ct_test_rest_call( 'ct_scan_admit', '/ct-scan/v1/admit', $payload ); +} + +/** Make the previous admission old enough that pacing allows the next one. */ +function ct_test_age_admission( $job_id, $ms = 1100 ) { + global $wpdb; + $wpdb->query( $wpdb->prepare( 'UPDATE ' . ct_scan_jobs_table() . ' SET last_admit_ms = last_admit_ms - %d WHERE job_id = %d', $ms, $job_id ) ); +} + +/** Remove only leftovers of earlier fixture runs. */ +function ct_test_purge_fixtures() { + global $wpdb; + $jobs = ct_scan_jobs_table(); + $old_fixture_ids = $wpdb->get_col( $wpdb->prepare( "SELECT request_id FROM {$jobs} WHERE contact_name = %s", 'Fixture Reviewer' ) ); + foreach ( (array) $old_fixture_ids as $old_id ) { + $old_post = get_post( (int) $old_id ); + if ( $old_post && 'ct_promo_request' === $old_post->post_type && 'private' === $old_post->post_status && 'Integration fixture request' === $old_post->post_title ) { + $wpdb->delete( $jobs, array( 'request_id' => (int) $old_id ), array( '%d' ) ); + wp_delete_post( (int) $old_id, true ); + } + } +} diff --git a/integrations/worker/admission.py b/integrations/worker/admission.py new file mode 100644 index 0000000..2209630 --- /dev/null +++ b/integrations/worker/admission.py @@ -0,0 +1,339 @@ +"""Per-request admission guard. + +The sandboxed scanner holds no worker key and no authorization state. Before +each connection to the customer's site it asks the root supervisor over a +private socket; the supervisor asks WordPress (signed ``admit`` route), which +atomically checks cancellation, expiry, lease and budget under the queue lock. + +Fail-closed rules: + +* No grant, no connection. Any error, timeout, malformed answer, protocol + violation or unavailable store is a refusal. +* A grant is reserved and consumed in one step, is valid for at most + ``MAX_VALID_SECONDS`` and covers exactly one connection and request. +* Only one grant may be outstanding. The supervisor kills the scanner if a + request is not reported finished within ``MAX_REQUEST_SECONDS`` (plus a short + grace), if the scanner skips the protocol, or if WordPress reports that the + job is no longer live. +""" + +from __future__ import annotations + +import json +import re +import time +from dataclasses import dataclass + +from .protocol import APIError + + +MAX_VALID_SECONDS = 1.0 +MAX_REQUEST_SECONDS = 10.0 +REQUEST_GRACE_SECONDS = 1.0 +PROBE_INTERVAL_SECONDS = 2.0 +MAX_ADMIT_WAIT_SECONDS = 4.0 +BUSY_RETRY_SECONDS = 0.15 +MAX_MESSAGE_BYTES = 256 +CHILD_REPLY_TIMEOUT_SECONDS = 15.0 +REASON_RE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") +RETRYABLE_BUSY = {"queue_busy", "admission_write_failed"} + + +class AdmissionDenied(Exception): + """A refusal with a log-safe reason. Never converted into permission.""" + + def __init__(self, slug: str, *, retry_after_ms: int = 0): + super().__init__(slug) + self.slug = slug if REASON_RE.fullmatch(slug or "") else "admission_denied" + self.retry_after_ms = retry_after_ms + + +@dataclass(frozen=True) +class Grant: + sequence: int + valid_seconds: float + timeout_seconds: float + remaining: int + + +def _is_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def parse_grant(value) -> Grant: + """Strictly validate a WordPress grant and clamp it to the disclosed hard bounds.""" + expected = {"ok", "admitted", "sequence", "valid_ms", "timeout_ms", "http_remaining"} + if not isinstance(value, dict) or set(value) != expected or value["ok"] is not True or value["admitted"] is not True: + raise AdmissionDenied("admission_response_invalid") + sequence, valid_ms, timeout_ms, remaining = (value[k] for k in ("sequence", "valid_ms", "timeout_ms", "http_remaining")) + if not all(_is_int(item) for item in (sequence, valid_ms, timeout_ms, remaining)): + raise AdmissionDenied("admission_response_invalid") + if sequence < 1 or valid_ms < 1 or timeout_ms < 1 or remaining < 0: + raise AdmissionDenied("admission_response_invalid") + return Grant( + sequence, + min(valid_ms / 1000.0, MAX_VALID_SECONDS), + min(timeout_ms / 1000.0, MAX_REQUEST_SECONDS), + remaining, + ) + + +def parse_refusal(error: APIError) -> AdmissionDenied: + """Turn a non-2xx answer into a refusal; unknown or malformed answers stay refusals.""" + payload = getattr(error, "payload", None) + reason = payload.get("reason") if isinstance(payload, dict) else None + if not isinstance(reason, str) or not REASON_RE.fullmatch(reason): + reason = "guard_unavailable" + retry = payload.get("retry_after_ms") if isinstance(payload, dict) else 0 + if not _is_int(retry) or retry < 0: + retry = 0 + return AdmissionDenied(reason, retry_after_ms=min(retry, 1000)) + + +class AdmissionGate: + """Supervisor-side client for the WordPress admission route.""" + + def __init__(self, api, claim, *, stop_event=None, clock=time.time, monotonic=time.monotonic): + self.api = api + self.claim = claim + self.stop_event = stop_event + self._clock = clock + self._monotonic = monotonic + + def _wait(self, seconds: float): + if self.stop_event is not None: + if self.stop_event.wait(seconds): + raise AdmissionDenied("worker_shutdown") + else: + time.sleep(seconds) + + def _check_local(self): + if self.stop_event is not None and self.stop_event.is_set(): + raise AdmissionDenied("worker_shutdown") + if self._clock() >= self.claim.expires_at.timestamp(): + raise AdmissionDenied("authorization_expired") + + def _call(self, kind: str): + try: + response = self.api.admit(self.claim, kind) + except APIError as exc: + if exc.status is None: + raise AdmissionDenied("guard_unavailable") from exc + raise parse_refusal(exc) from exc + return response + + def admit(self) -> Grant: + """Reserve exactly one request, or raise AdmissionDenied. Retries are bounded.""" + deadline = self._monotonic() + MAX_ADMIT_WAIT_SECONDS + while True: + self._check_local() + try: + return parse_grant(self._call("http")) + except AdmissionDenied as denied: + if self._monotonic() >= deadline: + raise + if denied.slug == "rate_wait" and denied.retry_after_ms > 0: + self._wait(min(denied.retry_after_ms / 1000.0, 1.0)) + elif denied.slug in RETRYABLE_BUSY: + self._wait(BUSY_RETRY_SECONDS) + else: + raise + + def probe(self): + """Ask whether the job is still live. Consumes no budget and does not pace.""" + self._check_local() + response = self._call("probe") + if not isinstance(response, dict) or set(response) != {"ok", "live"} or response["ok"] is not True or response["live"] is not True: + raise AdmissionDenied("admission_response_invalid") + + +class AdmissionPump: + """Supervisor side of the scanner socket: a one-grant-at-a-time state machine. + + ``service`` and ``tick`` return a fatal reason slug when the scanner must be + killed, and ``None`` otherwise. After a fatal reason the pump answers nothing. + """ + + def __init__(self, sock, gate, *, monotonic=time.monotonic): + self.sock = sock + self.gate = gate + self._monotonic = monotonic + self._buffer = bytearray() + self._inflight_until = None + self._last_probe = monotonic() + self.grants = 0 + self.fatal = None + self.closed = False + + @property + def inflight(self) -> bool: + return self._inflight_until is not None + + def _send(self, value) -> bool: + data = json.dumps(value, separators=(",", ":")).encode("ascii") + b"\n" + try: + self.sock.settimeout(2.0) + self.sock.sendall(data) + return True + except OSError: + return False + + def _fail(self, slug: str): + slug = slug if slug.startswith(("admission_", "request_")) else "admission_" + slug + self.fatal = slug[:64] if REASON_RE.fullmatch(slug[:64]) else "admission_denied" + return self.fatal + + def _handle(self, line: bytes): + try: + message = json.loads(line.decode("ascii")) + except (UnicodeDecodeError, json.JSONDecodeError): + return self._fail("admission_protocol_violation") + if not isinstance(message, dict): + return self._fail("admission_protocol_violation") + if message == {"op": "done"}: + if not self.inflight: + return self._fail("admission_protocol_violation") + self._inflight_until = None + return None + if message == {"op": "admit", "kind": "http"}: + if self.inflight: + # One connection at a time: a second grant is never available. + return self._fail("admission_protocol_violation") + try: + grant = self.gate.admit() + except AdmissionDenied as denied: + self._send({"ok": False, "reason": denied.slug}) + return self._fail(denied.slug) + # The deadline starts when WordPress granted, before the reply is delivered. + self._inflight_until = self._monotonic() + grant.timeout_seconds + REQUEST_GRACE_SECONDS + self.grants += 1 + if not self._send({"ok": True, "valid_ms": int(grant.valid_seconds * 1000)}): + return self._fail("admission_channel_failed") + return None + return self._fail("admission_protocol_violation") + + def service(self): + """Read and handle whatever the scanner has sent. Call when the socket is readable.""" + if self.fatal: + return self.fatal + try: + self.sock.settimeout(0) + chunk = self.sock.recv(MAX_MESSAGE_BYTES + 1) + except (BlockingIOError, InterruptedError): + return None + except OSError: + self.closed = True + return None + if not chunk: + self.closed = True + return None + self._buffer.extend(chunk) + while b"\n" in self._buffer: + line, _, rest = bytes(self._buffer).partition(b"\n") + self._buffer = bytearray(rest) + if len(line) > MAX_MESSAGE_BYTES: + return self._fail("admission_protocol_violation") + fatal = self._handle(line) + if fatal: + return fatal + if len(self._buffer) > MAX_MESSAGE_BYTES: + return self._fail("admission_protocol_violation") + return None + + def tick(self): + """Enforce the in-flight bound and watch for cancellation, expiry or revocation.""" + if self.fatal: + return self.fatal + now = self._monotonic() + if self._inflight_until is not None and now > self._inflight_until: + return self._fail("request_deadline_exceeded") + if now - self._last_probe >= PROBE_INTERVAL_SECONDS: + self._last_probe = now + try: + self.gate.probe() + except AdmissionDenied as denied: + # An unreachable guard does not abort the request already admitted (its own + # ten-second bound applies), and the next admission fails closed. An explicit + # refusal stops the scanner now. + if denied.slug not in {"guard_unavailable", "admission_response_invalid"}: + return self._fail(denied.slug) + return None + + +class AdmissionRefused(OSError): + """Raised inside the scanner when no admission was granted. Subclasses OSError so + that an unmodified checker treats it as a failed fetch and sends nothing.""" + + +class ChildAdmission: + """Scanner-side client of the supervisor socket. Holds no key and no state of record.""" + + def __init__(self, sock, *, monotonic=time.monotonic): + self.sock = sock + self._monotonic = monotonic + self._granted_at = None + self._valid_seconds = 0.0 + self._outstanding = False + + def _read_line(self) -> bytes: + deadline = self._monotonic() + CHILD_REPLY_TIMEOUT_SECONDS + data = bytearray() + while True: + remaining = deadline - self._monotonic() + if remaining <= 0: + raise AdmissionRefused("admission_timeout") + self.sock.settimeout(remaining) + try: + chunk = self.sock.recv(1) + except OSError as exc: + raise AdmissionRefused("admission_channel_failed") from exc + if not chunk: + raise AdmissionRefused("admission_channel_closed") + if chunk == b"\n": + return bytes(data) + data.extend(chunk) + if len(data) > MAX_MESSAGE_BYTES: + raise AdmissionRefused("admission_reply_invalid") + + def admit(self): + if self._outstanding: + raise AdmissionRefused("admission_outstanding") + try: + self.sock.settimeout(5.0) + self.sock.sendall(b'{"op":"admit","kind":"http"}\n') + except OSError as exc: + raise AdmissionRefused("admission_channel_failed") from exc + try: + reply = json.loads(self._read_line().decode("ascii")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AdmissionRefused("admission_reply_invalid") from exc + if not isinstance(reply, dict): + raise AdmissionRefused("admission_reply_invalid") + if reply.get("ok") is not True: + reason = reply.get("reason") + raise AdmissionRefused(reason if isinstance(reason, str) and REASON_RE.fullmatch(reason) else "admission_denied") + valid_ms = reply.get("valid_ms") + if set(reply) != {"ok", "valid_ms"} or not _is_int(valid_ms) or not 1 <= valid_ms <= int(MAX_VALID_SECONDS * 1000): + raise AdmissionRefused("admission_reply_invalid") + self._granted_at = self._monotonic() + self._valid_seconds = valid_ms / 1000.0 + self._outstanding = True + + def ensure_fresh(self): + """Call immediately before connecting: a stale grant is never used.""" + if not self._outstanding or self._granted_at is None: + raise AdmissionRefused("admission_missing") + if self._monotonic() - self._granted_at >= self._valid_seconds: + raise AdmissionRefused("admission_expired") + + def done(self): + """Report the request finished (success or failure). Idempotent.""" + if not self._outstanding: + return + self._outstanding = False + self._granted_at = None + try: + self.sock.settimeout(2.0) + self.sock.sendall(b'{"op":"done"}\n') + except OSError: + pass diff --git a/integrations/worker/network.py b/integrations/worker/network.py index d59af03..7ebe0b8 100644 --- a/integrations/worker/network.py +++ b/integrations/worker/network.py @@ -206,26 +206,32 @@ def _connect_socket(family, address: str, port: int, timeout, source_address=Non class PinnedHTTPConnection(http.client.HTTPConnection): - def __init__(self, host, port, *, pinned_family, pinned_address, timeout=None, **kwargs): + def __init__(self, host, port, *, pinned_family, pinned_address, timeout=None, before_connect=None, **kwargs): super().__init__(host, port, timeout=timeout, **kwargs) self._pinned_family = pinned_family self._pinned_address = pinned_address + self._before_connect = before_connect def connect(self): + if self._before_connect is not None: + self._before_connect() self.sock = _connect_socket( self._pinned_family, self._pinned_address, self.port, self.timeout, self.source_address ) class PinnedHTTPSConnection(http.client.HTTPSConnection): - def __init__(self, host, port, *, pinned_family, pinned_address, timeout=None, context=None, **kwargs): + def __init__(self, host, port, *, pinned_family, pinned_address, timeout=None, context=None, before_connect=None, **kwargs): super().__init__(host, port, timeout=timeout, context=context or ssl.create_default_context(), **kwargs) self._pinned_family = pinned_family self._pinned_address = pinned_address + self._before_connect = before_connect def connect(self): if self._tunnel_host: raise OSError("tunnels are disabled") + if self._before_connect is not None: + self._before_connect() raw = _connect_socket( self._pinned_family, self._pinned_address, self.port, self.timeout, self.source_address ) @@ -238,9 +244,10 @@ def connect(self): class _ManagedResponse: - def __init__(self, response, connection): + def __init__(self, response, connection, on_close=None): self._response = response self._connection = connection + self._on_close = on_close self.code = response.status self.headers = response.headers @@ -251,7 +258,11 @@ def close(self): try: self._response.close() finally: - self._connection.close() + try: + self._connection.close() + finally: + if self._on_close is not None: + self._on_close() def __enter__(self): return self @@ -261,12 +272,19 @@ def __exit__(self, *_exc): class PinnedOpener: - """urllib-compatible single request transport with no proxies or auto-redirects.""" + """urllib-compatible single request transport with no proxies or auto-redirects. - def __init__(self, scope: RequestOrigin, *, resolver=socket.getaddrinfo, ssl_context=None): + With an admission ``guard`` (the scanner child), every request first obtains a + one-request admission from the supervisor, re-checks that it is still fresh + immediately before connecting, and reports completion. Without a guard (the + fixed WordPress API client and offline tests) no target is contacted. + """ + + def __init__(self, scope: RequestOrigin, *, resolver=socket.getaddrinfo, ssl_context=None, guard=None): self.scope = scope self.resolver = resolver self.ssl_context = ssl_context or ssl.create_default_context() + self.guard = guard def open(self, request, timeout=None): url = request.full_url if hasattr(request, "full_url") else str(request) @@ -275,19 +293,30 @@ def open(self, request, timeout=None): scheme, port, host, target = self.scope.validate_request(url) answers = _public_addresses(host, port, self.resolver) family, address = answers[0] - connection_type = PinnedHTTPSConnection if scheme == "https" else PinnedHTTPConnection - kwargs = {"pinned_family": family, "pinned_address": address, "timeout": timeout} - if scheme == "https": - kwargs["context"] = self.ssl_context - connection = connection_type(host, port, **kwargs) - headers = dict(request.header_items()) if hasattr(request, "header_items") else {} - headers["Connection"] = "close" - method = request.get_method() if hasattr(request, "get_method") else "GET" - body = getattr(request, "data", None) + guard = self.guard + if guard is not None: + # No grant, no connection. Raises before any socket is opened. + guard.admit() try: - connection.request(method, target, body=body, headers=headers) - response = connection.getresponse() - return _ManagedResponse(response, connection) + connection_type = PinnedHTTPSConnection if scheme == "https" else PinnedHTTPConnection + kwargs = {"pinned_family": family, "pinned_address": address, "timeout": timeout} + if guard is not None: + kwargs["before_connect"] = guard.ensure_fresh + if scheme == "https": + kwargs["context"] = self.ssl_context + connection = connection_type(host, port, **kwargs) + headers = dict(request.header_items()) if hasattr(request, "header_items") else {} + headers["Connection"] = "close" + method = request.get_method() if hasattr(request, "get_method") else "GET" + body = getattr(request, "data", None) + try: + connection.request(method, target, body=body, headers=headers) + response = connection.getresponse() + return _ManagedResponse(response, connection, guard.done if guard is not None else None) + except BaseException: + connection.close() + raise except BaseException: - connection.close() + if guard is not None: + guard.done() raise diff --git a/integrations/worker/protocol.py b/integrations/worker/protocol.py index 8093aef..e0eb409 100644 --- a/integrations/worker/protocol.py +++ b/integrations/worker/protocol.py @@ -21,6 +21,9 @@ API_HOST = "complete.tech" API_BASE_PATH = "/wp-json/ct-scan/v1" AUTH_ROUTE_PREFIX = "/ct-scan/v1" +ROUTES = frozenset( + f"{AUTH_ROUTE_PREFIX}/{name}" for name in ("claim", "start", "admit", "result") +) API_MAX_BODY = 64 * 1024 API_MAX_RESPONSE = 16 * 1024 MAX_JOB_ID = 2**63 - 1 @@ -33,10 +36,12 @@ class APIError(Exception): """A transport or protocol failure with a log-safe category.""" - def __init__(self, slug: str, *, status: int | None = None): + def __init__(self, slug: str, *, status: int | None = None, payload=None): super().__init__(slug) self.slug = slug self.status = status + # Parsed JSON object of a non-2xx answer, when one was returned (never logged). + self.payload = payload @dataclass(frozen=True) @@ -51,7 +56,7 @@ class JobClaim: def auth_headers(secret: bytes, timestamp: int, nonce: str, route: str, raw_body: bytes) -> dict[str, str]: - if route not in {f"{AUTH_ROUTE_PREFIX}/claim", f"{AUTH_ROUTE_PREFIX}/start", f"{AUTH_ROUTE_PREFIX}/result"}: + if route not in ROUTES: raise ValueError("invalid_route") timestamp_text = str(timestamp) body_hash = hashlib.sha256(raw_body).hexdigest() @@ -129,15 +134,15 @@ def __init__(self, secret: bytes, *, transport=None, clock=time.time, nonce_fact self._nonce_factory = nonce_factory or (lambda: secrets.token_hex(16)) self._transport = transport or PinnedOpener(RequestOrigin(API_HOST, "https", 443)) - def post(self, route: str, payload): + def post(self, route: str, payload, *, timeout: float = 10.0): try: raw_body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") except (TypeError, ValueError, UnicodeError) as exc: raise APIError("invalid_api_payload") from exc - return self.post_raw(route, raw_body) + return self.post_raw(route, raw_body, timeout=timeout) - def post_raw(self, route: str, raw_body: bytes): - if route not in {"/ct-scan/v1/claim", "/ct-scan/v1/start", "/ct-scan/v1/result"}: + def post_raw(self, route: str, raw_body: bytes, *, timeout: float = 10.0): + if route not in ROUTES: raise APIError("invalid_route") if not isinstance(raw_body, bytes) or len(raw_body) > API_MAX_BODY: raise APIError("api_payload_too_large") @@ -149,7 +154,7 @@ def post_raw(self, route: str, raw_body: bytes): full_url = API_SITE_URL + "/wp-json" + route request = Request(full_url, data=raw_body, headers=headers, method="POST") try: - response = self._transport.open(request, timeout=10.0) + response = self._transport.open(request, timeout=timeout) except (http.client.HTTPException, OSError, URLError, TimeoutError, ValueError) as exc: raise APIError("api_network_error") from exc try: @@ -160,12 +165,12 @@ def post_raw(self, route: str, raw_body: bytes): raise APIError("api_network_error") from exc if len(raw_response) > API_MAX_RESPONSE: raise APIError("api_response_too_large", status=status) - if not 200 <= status < 300: - raise APIError(f"api_http_{status}", status=status) try: decoded = json.loads(raw_response.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise APIError("invalid_api_response", status=status) from exc + except (UnicodeDecodeError, json.JSONDecodeError): + decoded = None + if not 200 <= status < 300: + raise APIError(f"api_http_{status}", status=status, payload=decoded if isinstance(decoded, dict) else None) if not isinstance(decoded, dict): raise APIError("invalid_api_response", status=status) return decoded @@ -181,6 +186,19 @@ def start(self, claim: JobClaim) -> bool: ) return type(response.get("ok")) is bool and response["ok"] is True + def admit(self, claim: JobClaim, kind: str) -> dict: + """Ask WordPress to admit one request ("http") or report liveness ("probe"). + + A short timeout keeps a slow store from stalling the supervisor; a timeout is a refusal. + """ + if kind not in {"http", "probe"}: + raise APIError("invalid_admission_kind") + return self.post( + "/ct-scan/v1/admit", + {"id": claim.job_id, "lease": claim.lease, "scope_hash": claim.scope_hash, "kind": kind}, + timeout=5.0, + ) + def result(self, body: bytes) -> dict: response = self.post_raw("/ct-scan/v1/result", body) return response diff --git a/integrations/worker/runner.py b/integrations/worker/runner.py index 738c5c1..3f0690f 100644 --- a/integrations/worker/runner.py +++ b/integrations/worker/runner.py @@ -6,7 +6,9 @@ import os import platform import re +import select import signal +import socket import subprocess import sys import threading @@ -15,6 +17,7 @@ from wordpress_check import PHP2SHELL_CVES, PHP2SHELL_RULES_DATE, PHP2SHELL_SOURCES, Result, assess_php2shell +from .admission import AdmissionPump from .network import RequestOrigin @@ -110,7 +113,10 @@ def __init__(self, *, repo_root=None, popen=subprocess.Popen, monotonic=time.mon self._popen = popen self._monotonic = monotonic - def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None) -> dict: + def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None, gate=None) -> dict: + # Fail closed by construction: without an admission gate there is no scan. + if gate is None: + raise ScannerFailure("admission_guard_required") if not sys.platform.startswith("linux") or os.name != "posix" or os.geteuid() != 0: raise ScannerFailure("scanner_isolation_unavailable") if not _has_kill_capability(): @@ -135,6 +141,12 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None "PYTHONDONTWRITEBYTECODE": "1", "PYTHONPATH": self.repo_root, } + try: + parent_sock, child_sock = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + except OSError as exc: + raise ScannerFailure("admission_channel_unavailable") from exc + child_fd = child_sock.fileno() + env["CT_ADMISSION_FD"] = str(child_fd) command = [sys.executable, "-m", "integrations.worker.scanner"] try: process = self._popen( @@ -145,37 +157,86 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, + pass_fds=(child_fd,), start_new_session=True, preexec_fn=_linux_preexec_limits, bufsize=0, ) except (OSError, ValueError) as exc: + parent_sock.close() + child_sock.close() raise ScannerFailure("scanner_start_failed") from exc + child_sock.close() + pump = AdmissionPump(parent_sock, gate) - buffers = {"stdout": bytearray(), "stderr": bytearray()} - overflow = threading.Event() - stdout_reader = threading.Thread( - target=_read_bounded, - args=(process.stdout, MAX_SCANNER_STDOUT, buffers, "stdout", overflow), - daemon=True, - ) - stderr_reader = threading.Thread( - target=_read_bounded, - args=(process.stderr, MAX_SCANNER_STDERR, buffers, "stderr", overflow), - daemon=True, - ) - stdout_reader.start() - stderr_reader.start() try: - view = memoryview(payload) - while view: - written = process.stdin.write(view) - if not written: - raise OSError("short scanner input write") - view = view[written:] - process.stdin.close() - except OSError: - self._kill_process(process) + buffers = {"stdout": bytearray(), "stderr": bytearray()} + overflow = threading.Event() + stdout_reader = threading.Thread( + target=_read_bounded, + args=(process.stdout, MAX_SCANNER_STDOUT, buffers, "stdout", overflow), + daemon=True, + ) + stderr_reader = threading.Thread( + target=_read_bounded, + args=(process.stderr, MAX_SCANNER_STDERR, buffers, "stderr", overflow), + daemon=True, + ) + stdout_reader.start() + stderr_reader.start() + try: + view = memoryview(payload) + while view: + written = process.stdin.write(view) + if not written: + raise OSError("short scanner input write") + view = view[written:] + process.stdin.close() + except OSError: + self._kill_process(process) + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + self._kill_process(process) + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired as exc: + _close_pipes(process) + raise ScannerFailure("scanner_termination_failed") from exc + stdout_reader.join(timeout=2) + stderr_reader.join(timeout=2) + _close_pipes(process) + raise ScannerFailure("scanner_input_failed") + + deadline = self._monotonic() + float(timeout) + timed_out = False + stopped = False + while process.poll() is None: + if overflow.is_set(): + self._kill_process(process) + break + if stop_event is not None and stop_event.is_set(): + stopped = True + self._kill_process(process) + break + remaining = deadline - self._monotonic() + if remaining <= 0: + timed_out = True + self._kill_process(process) + break + fatal = None + if pump.closed: + time.sleep(min(0.025, remaining)) + else: + readable, _, _ = select.select([parent_sock], [], [], min(0.025, remaining)) + if readable: + fatal = pump.service() + # Admission, cancellation, expiry and the in-flight bound are enforced here, + # not by the scanner: any fatal reason kills it at once. + fatal = fatal or pump.tick() + if fatal: + self._kill_process(process) + break try: process.wait(timeout=2) except subprocess.TimeoutExpired: @@ -187,55 +248,28 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None raise ScannerFailure("scanner_termination_failed") from exc stdout_reader.join(timeout=2) stderr_reader.join(timeout=2) - _close_pipes(process) - raise ScannerFailure("scanner_input_failed") - - deadline = self._monotonic() + float(timeout) - timed_out = False - stopped = False - while process.poll() is None: - if overflow.is_set(): - self._kill_process(process) - break - if stop_event is not None and stop_event.is_set(): - stopped = True - self._kill_process(process) - break - remaining = deadline - self._monotonic() - if remaining <= 0: - timed_out = True - self._kill_process(process) - break - time.sleep(min(0.025, remaining)) - try: - process.wait(timeout=2) - except subprocess.TimeoutExpired: - self._kill_process(process) - try: - process.wait(timeout=2) - except subprocess.TimeoutExpired as exc: + if stdout_reader.is_alive() or stderr_reader.is_alive(): _close_pipes(process) - raise ScannerFailure("scanner_termination_failed") from exc - stdout_reader.join(timeout=2) - stderr_reader.join(timeout=2) - if stdout_reader.is_alive() or stderr_reader.is_alive(): - _close_pipes(process) - stdout_reader.join(timeout=2) - stderr_reader.join(timeout=2) - else: - _close_pipes(process) - if timed_out: - raise ScannerFailure("scanner_timeout") - if stopped: - raise ScannerFailure("worker_shutdown") - if overflow.is_set() or len(buffers["stdout"]) > MAX_SCANNER_STDOUT: - raise ScannerFailure("scanner_output_overflow") - if stdout_reader.is_alive() or stderr_reader.is_alive(): - raise ScannerFailure("scanner_pipe_stalled") - if process.returncode != 0: - child_error = self._parse_child_error(bytes(buffers["stdout"])) - raise ScannerFailure(child_error or "scanner_process_failed") - return self._parse_success(bytes(buffers["stdout"]), target, scope) + stdout_reader.join(timeout=2) + stderr_reader.join(timeout=2) + else: + _close_pipes(process) + if timed_out: + raise ScannerFailure("scanner_timeout") + if stopped: + raise ScannerFailure("worker_shutdown") + if pump.fatal: + raise ScannerFailure(pump.fatal) + if overflow.is_set() or len(buffers["stdout"]) > MAX_SCANNER_STDOUT: + raise ScannerFailure("scanner_output_overflow") + if stdout_reader.is_alive() or stderr_reader.is_alive(): + raise ScannerFailure("scanner_pipe_stalled") + if process.returncode != 0: + child_error = self._parse_child_error(bytes(buffers["stdout"])) + raise ScannerFailure(child_error or "scanner_process_failed") + return self._parse_success(bytes(buffers["stdout"]), target, scope) + finally: + parent_sock.close() @staticmethod def _kill_process(process): diff --git a/integrations/worker/scanner.py b/integrations/worker/scanner.py index 458f6a7..879e8ae 100644 --- a/integrations/worker/scanner.py +++ b/integrations/worker/scanner.py @@ -5,12 +5,14 @@ import json import os import signal +import socket import sys from dataclasses import asdict from wordpress_check import Checker +from .admission import ChildAdmission from .network import PinnedOpener, RequestOrigin @@ -37,6 +39,21 @@ def _write(value, exit_code: int) -> int: return exit_code +def _admission_channel(): + """Open the supervisor socket named by CT_ADMISSION_FD. Without it, nothing is contacted.""" + raw = os.environ.get("CT_ADMISSION_FD", "") + if not raw.isdigit() or not 3 <= int(raw) <= 1023: + return None + try: + channel = socket.socket(fileno=int(raw)) + except (OSError, ValueError): + return None + if channel.family != socket.AF_UNIX or channel.type != socket.SOCK_STREAM: + channel.close() + return None + return channel + + def _timeout(_signum, _frame): raise TimeoutError("scanner_timeout") @@ -59,8 +76,13 @@ def main() -> int: scope, target = RequestOrigin.from_target(data["target"]) if target != data["target"]: return _write({"ok": False, "error": "invalid_scanner_input"}, 2) + channel = _admission_channel() + if channel is None: + # Fail closed: a scanner that cannot ask for admission never sends a request. + return _write({"ok": False, "error": "admission_channel_unavailable"}, 2) + guard = ChildAdmission(channel) checker = Checker(timeout=10.0, delay=1.0, contact=CONTACT) - checker.opener = PinnedOpener(scope) + checker.opener = PinnedOpener(scope, guard=guard) result = checker.check(target) return _write({"ok": True, "result": asdict(result)}, 0) except TimeoutError: diff --git a/integrations/worker/service.py b/integrations/worker/service.py index 83e144b..e913105 100644 --- a/integrations/worker/service.py +++ b/integrations/worker/service.py @@ -16,6 +16,7 @@ from pathlib import Path from urllib.parse import urlsplit +from .admission import AdmissionGate from .network import RequestOrigin from .protocol import ( APIClient, @@ -256,7 +257,8 @@ def process_once(self): outcome, result, error = "failed", None, "stale_claim" else: try: - result = self.runner.run(claim.target, timeout=scan_timeout, stop_event=self.stop_event) + gate = AdmissionGate(self.api, claim, stop_event=self.stop_event, clock=self.clock) + result = self.runner.run(claim.target, timeout=scan_timeout, stop_event=self.stop_event, gate=gate) SubprocessRunner.validate_result(result, claim.target, scope) outcome, error = "completed", "" except ScannerFailure as exc: diff --git a/test_admission.py b/test_admission.py new file mode 100644 index 0000000..497c0d4 --- /dev/null +++ b/test_admission.py @@ -0,0 +1,833 @@ +"""Offline tests for the per-request admission guard. + +Everything here talks only to local disposable servers and socket pairs. No +external website is contacted. The supervisor tests run a real child process +and a local HTTP fixture; they patch the root/privilege checks (the isolation +itself is covered by the root-only tests in test_scan_worker.py). +""" + +from __future__ import annotations + +import hashlib +import http.server +import json +import os +import socket +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest import mock +from urllib.request import Request + +from integrations.worker import admission, network +from integrations.worker.admission import ( + AdmissionDenied, + AdmissionGate, + AdmissionPump, + AdmissionRefused, + ChildAdmission, + Grant, + parse_grant, + parse_refusal, +) +from integrations.worker.network import PinnedOpener, RequestOrigin +from integrations.worker.protocol import APIClient, APIError +from integrations.worker.runner import ScannerFailure, SubprocessRunner +from integrations.worker.service import WorkerService +from integrations.worker.spool import DurableSpool + +from test_scan_worker import REVISION, TARGET, _FakeAPI, _claim, _result + +NOW = 1_800_000_000 +GRANT = {"ok": True, "admitted": True, "sequence": 1, "valid_ms": 1000, "timeout_ms": 10000, "http_remaining": 19} +REPO_ROOT = Path(__file__).resolve().parent + + +def _refusal(status, reason, **extra): + return APIError(f"api_http_{status}", status=status, payload={"ok": False, "reason": reason, **extra}) + + +class _AdmitAPI: + """Scripted WordPress admission route.""" + + def __init__(self, *responses): + self.responses = list(responses) + self.calls = [] + + def admit(self, claim, kind): + self.calls.append(kind) + item = self.responses.pop(0) if self.responses else _refusal(410, "job_cancelled") + if isinstance(item, Exception): + raise item + return item + + +class _FakeClockEvent: + """Stand-in for threading.Event whose wait() advances a fake monotonic clock.""" + + def __init__(self, clock): + self.clock = clock + self.set_flag = False + + def is_set(self): + return self.set_flag + + def wait(self, seconds): + self.clock.now += seconds + return self.set_flag + + +class _Clock: + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + + +class GrantParsingTests(unittest.TestCase): + def test_valid_grant_is_clamped_to_the_disclosed_bounds(self): + grant = parse_grant(dict(GRANT, valid_ms=60000, timeout_ms=600000)) + self.assertEqual((grant.valid_seconds, grant.timeout_seconds, grant.sequence, grant.remaining), (1.0, 10.0, 1, 19)) + + def test_malformed_grants_are_refusals(self): + bad = [ + None, + [], + {}, + dict(GRANT, ok=False), + dict(GRANT, admitted=False), + dict(GRANT, ok=1), + dict(GRANT, extra=True), + {k: v for k, v in GRANT.items() if k != "sequence"}, + dict(GRANT, sequence=True), + dict(GRANT, sequence="1"), + dict(GRANT, sequence=0), + dict(GRANT, valid_ms=0), + dict(GRANT, timeout_ms=-5), + dict(GRANT, http_remaining=-1), + dict(GRANT, valid_ms=1.5), + ] + for value in bad: + with self.subTest(value=value), self.assertRaises(AdmissionDenied): + parse_grant(value) + + def test_refusal_reasons_are_sanitized_and_default_to_unavailable(self): + self.assertEqual(parse_refusal(_refusal(410, "job_cancelled")).slug, "job_cancelled") + self.assertEqual(parse_refusal(APIError("api_http_500", status=500)).slug, "guard_unavailable") + self.assertEqual(parse_refusal(_refusal(410, "Bad Reason; DROP")).slug, "guard_unavailable") + self.assertEqual(parse_refusal(_refusal(429, "rate_wait", retry_after_ms=99999)).retry_after_ms, 1000) + self.assertEqual(parse_refusal(_refusal(429, "rate_wait", retry_after_ms="soon")).retry_after_ms, 0) + + +class GateTests(unittest.TestCase): + def setUp(self): + self.clock = _Clock() + self.stop = _FakeClockEvent(self.clock) + + def gate(self, *responses, claim=None, wall=None): + self.api = _AdmitAPI(*responses) + return AdmissionGate( + self.api, + claim or _claim(expires_at=NOW + 7200), + stop_event=self.stop, + clock=wall or (lambda: NOW), + monotonic=self.clock, + ) + + def test_grant_is_returned_for_exactly_one_call(self): + grant = self.gate(GRANT).admit() + self.assertIsInstance(grant, Grant) + self.assertEqual(self.api.calls, ["http"]) + + def test_cancellation_and_every_other_refusal_stop_immediately_without_retry(self): + for reason in ("job_cancelled", "job_expired", "authorization_revoked_or_expired", "budget_exhausted", "lease_invalid_or_expired", "profile_unknown", "recipient_unconfirmed", "clock_anomaly"): + with self.subTest(reason=reason): + gate = self.gate(_refusal(410, reason), GRANT) + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + self.assertEqual(denied.exception.slug, reason) + self.assertEqual(self.api.calls, ["http"], "a refusal must not be retried into a grant") + + def test_unavailable_store_or_garbled_answers_fail_closed(self): + cases = [ + APIError("api_network_error"), + APIError("api_http_500", status=500), + APIError("api_http_503", status=503, payload="not a dict"), + APIError("invalid_api_response", status=200), + {"ok": True}, + ] + for case in cases: + with self.subTest(case=repr(case)): + with self.assertRaises(AdmissionDenied): + self.gate(case).admit() + + def test_pacing_waits_are_bounded_and_then_grant(self): + grant = self.gate(_refusal(429, "rate_wait", retry_after_ms=400), _refusal(429, "rate_wait", retry_after_ms=300), GRANT).admit() + self.assertEqual(self.api.calls, ["http", "http", "http"]) + self.assertAlmostEqual(self.clock.now - 1000.0, 0.7, places=3) + self.assertEqual(grant.sequence, 1) + + def test_endless_pacing_or_busy_answers_end_in_a_refusal(self): + for refusal in (_refusal(429, "rate_wait", retry_after_ms=1000), _refusal(503, "queue_busy")): + with self.subTest(refusal=refusal.payload["reason"]): + self.clock.now = 1000.0 + gate = self.gate(*([refusal] * 100)) + with self.assertRaises(AdmissionDenied): + gate.admit() + self.assertLess(len(self.api.calls), 100) + self.assertLessEqual(self.clock.now - 1000.0, admission.MAX_ADMIT_WAIT_SECONDS + 1.1) + + def test_expired_authorization_is_refused_locally_without_asking(self): + gate = self.gate(GRANT, claim=_claim(expires_at=NOW - 1), wall=lambda: NOW) + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + self.assertEqual(denied.exception.slug, "authorization_expired") + self.assertEqual(self.api.calls, []) + + def test_shutdown_is_refused_locally_and_interrupts_waits(self): + gate = self.gate(GRANT) + self.stop.set_flag = True + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + self.assertEqual(denied.exception.slug, "worker_shutdown") + self.assertEqual(self.api.calls, []) + + def test_probe_requires_an_exact_live_answer(self): + self.gate({"ok": True, "live": True}).probe() + self.assertEqual(self.api.calls, ["probe"]) + for bad in ({"ok": True}, {"ok": True, "live": False}, {"ok": True, "live": True, "x": 1}, {"ok": False, "live": True}): + with self.subTest(bad=bad), self.assertRaises(AdmissionDenied): + self.gate(bad).probe() + with self.assertRaises(AdmissionDenied) as denied: + self.gate(_refusal(410, "job_cancelled")).probe() + self.assertEqual(denied.exception.slug, "job_cancelled") + + +class ApiClientAdmitTests(unittest.TestCase): + def test_admit_is_signed_for_its_own_route_with_a_short_timeout_and_exact_body(self): + captured = {} + + class Transport: + def open(self, request, timeout): + captured["url"] = request.full_url + captured["body"] = request.data + captured["timeout"] = timeout + captured["headers"] = dict(request.header_items()) + from test_scan_worker import _Response + + return _Response(json.dumps(GRANT).encode()) + + client = APIClient(b"s" * 32, transport=Transport(), clock=lambda: NOW, nonce_factory=lambda: "0" * 32) + claim = _claim() + self.assertEqual(client.admit(claim, "http"), GRANT) + self.assertEqual(captured["url"], "https://complete.tech/wp-json/ct-scan/v1/admit") + self.assertEqual(captured["timeout"], 5.0) + self.assertEqual(json.loads(captured["body"]), {"id": claim.job_id, "lease": claim.lease, "scope_hash": claim.scope_hash, "kind": "http"}) + body_hash = hashlib.sha256(captured["body"]).hexdigest() + import hmac + + canonical = f"{NOW}\n{'0' * 32}\nPOST\n/ct-scan/v1/admit\n{body_hash}".encode() + self.assertEqual(captured["headers"]["X-ct-signature"], hmac.new(b"s" * 32, canonical, hashlib.sha256).hexdigest()) + + def test_unknown_kinds_are_not_sent(self): + client = APIClient(b"s" * 32, transport=mock.Mock()) + for kind in ("tls", "", None, "HTTP"): + with self.subTest(kind=kind), self.assertRaises(APIError): + client.admit(_claim(), kind) + + def test_non_success_answers_carry_the_parsed_refusal(self): + class Transport: + def open(self, request, timeout): + from test_scan_worker import _Response + + return _Response(b'{"ok":false,"reason":"job_cancelled"}', code=410) + + client = APIClient(b"s" * 32, transport=Transport(), clock=lambda: NOW) + with self.assertRaises(APIError) as error: + client.admit(_claim(), "probe") + self.assertEqual((error.exception.status, error.exception.payload), (410, {"ok": False, "reason": "job_cancelled"})) + + +class _PumpHarness: + """Run the child end on a thread so the blocking request/reply protocol can be exercised.""" + + def __init__(self, test, gate, monotonic=time.monotonic): + self.parent, child = socket.socketpair() + self.child = ChildAdmission(child, monotonic=monotonic) + self.child_sock = child + self.pump = AdmissionPump(self.parent, gate, monotonic=monotonic) + test.addCleanup(self.parent.close) + test.addCleanup(child.close) + + def child_admit(self): + outcome = {} + + def run(): + try: + self.child.admit() + outcome["ok"] = True + except AdmissionRefused as exc: + outcome["refused"] = str(exc) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + deadline = time.monotonic() + 3 + fatal = None + while thread.is_alive() and time.monotonic() < deadline: + readable = __import__("select").select([self.parent], [], [], 0.05)[0] + if readable: + fatal = self.pump.service() + thread.join(2) + return outcome, fatal + + +class _ScriptGate: + def __init__(self, admit=None, probe=None): + self._admit = list(admit or []) + self._probe = list(probe or []) + self.admits = 0 + self.probes = 0 + + def admit(self): + self.admits += 1 + item = self._admit.pop(0) if self._admit else AdmissionDenied("job_cancelled") + if isinstance(item, Exception): + raise item + return item + + def probe(self): + self.probes += 1 + item = self._probe.pop(0) if self._probe else None + if isinstance(item, Exception): + raise item + + +def _grant(timeout=10.0): + return Grant(1, 1.0, timeout, 19) + + +class PumpStateMachineTests(unittest.TestCase): + def test_admit_then_done_allows_the_next_admission(self): + gate = _ScriptGate(admit=[_grant(), _grant()]) + harness = _PumpHarness(self, gate) + outcome, fatal = harness.child_admit() + self.assertEqual((outcome, fatal), ({"ok": True}, None)) + self.assertTrue(harness.pump.inflight) + harness.child.ensure_fresh() + harness.child.done() + harness.pump.service() if __import__("select").select([harness.parent], [], [], 1)[0] else None + self.assertFalse(harness.pump.inflight) + outcome, fatal = harness.child_admit() + self.assertEqual((outcome, fatal, gate.admits), ({"ok": True}, None, 2)) + + def test_a_second_admission_while_one_is_outstanding_kills_the_scanner(self): + harness = _PumpHarness(self, _ScriptGate(admit=[_grant(), _grant()])) + self.assertEqual(harness.child_admit()[0], {"ok": True}) + harness.child_sock.sendall(b'{"op":"admit","kind":"http"}\n') + self.assertTrue(__import__("select").select([harness.parent], [], [], 1)[0]) + self.assertEqual(harness.pump.service(), "admission_protocol_violation") + self.assertEqual(harness.pump.grants, 1) + + def test_malformed_messages_are_protocol_violations(self): + for raw in (b"done\n", b'{"op":"done"}\n', b'{"op":"admit"}\n', b'{"op":"admit","kind":"tls"}\n', b'{"op":"admit","kind":"http","x":1}\n', b"[]\n", b"\xff\n", b"x" * 400 + b"\n", b"x" * 300): + with self.subTest(raw=raw[:20]): + harness = _PumpHarness(self, _ScriptGate(admit=[_grant()])) + harness.child_sock.sendall(raw) + __import__("select").select([harness.parent], [], [], 1) + self.assertEqual(harness.pump.service(), "admission_protocol_violation") + self.assertEqual(harness.pump.grants, 0) + + def test_refusal_is_delivered_to_the_scanner_and_is_fatal(self): + gate = _ScriptGate(admit=[AdmissionDenied("job_cancelled")]) + harness = _PumpHarness(self, gate) + outcome, fatal = harness.child_admit() + self.assertEqual(outcome, {"refused": "job_cancelled"}) + self.assertEqual(fatal, "admission_job_cancelled") + self.assertEqual(harness.pump.service(), "admission_job_cancelled", "a fatal pump stays fatal") + + def test_in_flight_deadline_is_enforced_by_the_supervisor(self): + clock = _Clock() + gate = _ScriptGate(admit=[_grant(timeout=10.0)]) + harness = _PumpHarness(self, gate, monotonic=clock) + self.assertEqual(harness.child_admit()[0], {"ok": True}) + clock.now += 10.9 + self.assertIsNone(harness.pump.tick()) + clock.now += 0.2 + self.assertEqual(harness.pump.tick(), "request_deadline_exceeded") + + def test_probe_refusal_is_fatal_but_an_unreachable_guard_is_not_by_itself(self): + clock = _Clock() + gate = _ScriptGate(probe=[AdmissionDenied("guard_unavailable"), AdmissionDenied("job_cancelled")]) + harness = _PumpHarness(self, gate, monotonic=clock) + clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + self.assertIsNone(harness.pump.tick()) + self.assertEqual(gate.probes, 1) + self.assertIsNone(harness.pump.tick(), "probes are paced") + clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + self.assertEqual(harness.pump.tick(), "admission_job_cancelled") + + def test_unreachable_guard_fails_the_next_admission_closed(self): + class Down(_ScriptGate): + def admit(self): + self.admits += 1 + raise AdmissionDenied("guard_unavailable") + + harness = _PumpHarness(self, Down()) + outcome, fatal = harness.child_admit() + self.assertEqual((outcome, fatal), ({"refused": "guard_unavailable"}, "admission_guard_unavailable")) + + def test_peer_close_is_remembered_without_spinning(self): + harness = _PumpHarness(self, _ScriptGate()) + harness.child_sock.close() + self.assertIsNone(harness.pump.service()) + self.assertTrue(harness.pump.closed) + + +class ChildAdmissionTests(unittest.TestCase): + def setUp(self): + self.clock = _Clock() + self.parent, self.child = socket.socketpair() + self.addCleanup(self.parent.close) + self.addCleanup(self.child.close) + self.client = ChildAdmission(self.child, monotonic=self.clock) + + def reply(self, value): + threading.Thread(target=lambda: (self.parent.recv(256), self.parent.sendall(value)), daemon=True).start() + + def test_connect_is_refused_without_a_grant_and_after_the_grant_goes_stale(self): + with self.assertRaises(AdmissionRefused): + self.client.ensure_fresh() + self.reply(b'{"ok":true,"valid_ms":1000}\n') + self.client.admit() + self.clock.now += 0.5 + self.client.ensure_fresh() + self.clock.now += 0.6 + with self.assertRaises(AdmissionRefused) as stale: + self.client.ensure_fresh() + self.assertEqual(str(stale.exception), "admission_expired") + + def test_scanner_rejects_oversized_extended_or_garbled_replies(self): + for raw in (b'{"ok":true,"valid_ms":5000}\n', b'{"ok":true,"valid_ms":0}\n', b'{"ok":true,"valid_ms":1000,"x":1}\n', b'{"ok":true}\n', b'not json\n', b"[]\n", b"x" * 400 + b"\n"): + with self.subTest(raw=raw[:30]): + self.reply(raw) + with self.assertRaises(AdmissionRefused): + self.client.admit() + with self.assertRaises(AdmissionRefused): + self.client.ensure_fresh() + + def test_refusal_reason_is_surfaced_and_a_closed_channel_refuses(self): + self.reply(b'{"ok":false,"reason":"job_cancelled"}\n') + with self.assertRaises(AdmissionRefused) as refused: + self.client.admit() + self.assertEqual(str(refused.exception), "job_cancelled") + self.parent.close() + with self.assertRaises(AdmissionRefused): + self.client.admit() + + def test_a_second_admission_before_done_is_refused_in_the_scanner(self): + self.reply(b'{"ok":true,"valid_ms":1000}\n') + self.client.admit() + with self.assertRaises(AdmissionRefused): + self.client.admit() + self.client.done() + self.client.done() # idempotent + + +class _LocalScope: + def __init__(self, port): + self.port = port + + def validate_request(self, url): + from urllib.parse import urlsplit, urlunsplit + + parts = urlsplit(url) + if parts.scheme != "http" or parts.hostname != "example.test": + raise ValueError("request_out_of_scope") + return "http", self.port, "example.test", urlunsplit(("", "", parts.path or "/", parts.query, "")) + + +class _Recorder: + """Local disposable HTTP server that records every request it receives.""" + + def __init__(self, test, delay=0.0): + self.requests = [] + outer = self + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + outer.requests.append((time.monotonic(), self.path)) + if delay and self.path == "/slow": + time.sleep(delay) + body = b"fixture" + try: + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except OSError: + pass + + def log_message(self, *_args): + return + + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.port = self.server.server_port + thread = threading.Thread(target=self.server.serve_forever, daemon=True) + thread.start() + test.addCleanup(thread.join, 2) + test.addCleanup(self.server.server_close) + test.addCleanup(self.server.shutdown) + + +class _Guard: + """Child-side guard double with the same surface as ChildAdmission.""" + + def __init__(self, allow=1, stale=False): + self.allow = allow + self.stale = stale + self.events = [] + + def admit(self): + self.events.append("admit") + if self.allow <= 0: + raise AdmissionRefused("job_cancelled") + self.allow -= 1 + + def ensure_fresh(self): + self.events.append("fresh") + if self.stale: + raise AdmissionRefused("admission_expired") + + def done(self): + self.events.append("done") + + +class OpenerGuardTests(unittest.TestCase): + def open(self, guard, server): + opener = PinnedOpener(_LocalScope(server.port), guard=guard) + with mock.patch.object(network, "_public_addresses", return_value=[(socket.AF_INET, "127.0.0.1")]): + return opener.open(Request("http://example.test/page"), timeout=2) + + def test_a_refused_admission_opens_no_connection(self): + recorder = _Recorder(self) + guard = _Guard(allow=0) + with self.assertRaises(AdmissionRefused): + self.open(guard, recorder) + self.assertEqual(recorder.requests, []) + self.assertEqual(guard.events, ["admit"], "nothing to report done for a refused admission") + + def test_a_stale_grant_is_not_used_to_connect_and_is_released(self): + recorder = _Recorder(self) + guard = _Guard(stale=True) + with self.assertRaises(AdmissionRefused): + self.open(guard, recorder) + self.assertEqual(recorder.requests, []) + self.assertEqual(guard.events, ["admit", "fresh", "done"]) + + def test_one_grant_covers_one_request_and_done_follows_the_response(self): + recorder = _Recorder(self) + guard = _Guard(allow=2) + with self.open(guard, recorder) as response: + self.assertEqual(response.code, 200) + self.assertEqual(guard.events, ["admit", "fresh"]) + self.assertEqual(guard.events, ["admit", "fresh", "done"]) + self.assertEqual(len(recorder.requests), 1) + with self.open(guard, recorder): + pass + self.assertEqual(guard.events.count("admit"), 2) + self.assertEqual(len(recorder.requests), 2) + + def test_connection_failure_still_reports_done_once(self): + guard = _Guard() + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() # nothing listens here + opener = PinnedOpener(_LocalScope(port), guard=guard) + with mock.patch.object(network, "_public_addresses", return_value=[(socket.AF_INET, "127.0.0.1")]): + with self.assertRaises(OSError): + opener.open(Request("http://example.test/page"), timeout=2) + self.assertEqual(guard.events, ["admit", "fresh", "done"]) + + def test_dns_or_scope_failures_never_spend_an_admission(self): + guard = _Guard() + scope, _ = RequestOrigin.from_target(TARGET) + opener = PinnedOpener(scope, guard=guard) + with self.assertRaises(ValueError): + opener.open(Request("https://attacker.example.test/")) + with mock.patch.object(network, "_public_addresses", side_effect=ValueError("non_public_address")): + with self.assertRaises(ValueError): + opener.open(Request(TARGET)) + self.assertEqual(guard.events, []) + + def test_checker_sends_nothing_when_admission_is_refused(self): + from wordpress_check import Checker + + recorder = _Recorder(self) + checker = Checker(timeout=2, delay=0) + checker.opener = PinnedOpener(_LocalScope(recorder.port), guard=_Guard(allow=0)) + with mock.patch.object(network, "_public_addresses", return_value=[(socket.AF_INET, "127.0.0.1")]): + result = checker.check("http://example.test/") + self.assertEqual(result.status, "error") + self.assertEqual(recorder.requests, []) + + +class ScannerChannelTests(unittest.TestCase): + def test_scanner_has_no_channel_without_a_valid_unix_socket(self): + from integrations.worker import scanner + + with mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop("CT_ADMISSION_FD", None) + self.assertIsNone(scanner._admission_channel()) + for bad in ("", "x", "-1", "0", "2", "99999", "7.5"): + with self.subTest(bad=bad), mock.patch.dict(os.environ, {"CT_ADMISSION_FD": bad}): + self.assertIsNone(scanner._admission_channel()) + tcp = socket.socket() + self.addCleanup(tcp.close) + duplicate = os.dup(tcp.fileno()) + self.assertGreaterEqual(duplicate, 3) + with mock.patch.dict(os.environ, {"CT_ADMISSION_FD": str(duplicate)}): + self.assertIsNone(scanner._admission_channel(), "a TCP socket is not the supervisor channel") + + +# --- supervisor integration: a real child process, a local server, a flippable gate ----------- + +_CHILD_SCRIPT = r''' +import http.client, json, os, socket, sys, time +from urllib.parse import urlsplit +from integrations.worker.admission import ChildAdmission, AdmissionRefused + +PORT = {port} +RESULT = {result!r} +payload = json.loads(sys.stdin.buffer.read().decode("utf-8")) +host = urlsplit(payload["target"]).hostname +guard = ChildAdmission(socket.socket(fileno=int(os.environ["CT_ADMISSION_FD"]))) + +def fetch(path): + guard.admit() + try: + guard.ensure_fresh() + connection = http.client.HTTPConnection("127.0.0.1", PORT, timeout=10) + connection.request("GET", path) + connection.getresponse().read() + connection.close() + finally: + guard.done() + +def finish(): + result = json.loads(RESULT) + result["input_url"] = payload["target"] + result["final_url"] = payload["target"] + sys.stdout.write(json.dumps({{"ok": True, "result": result}}) + "\n") + raise SystemExit(0) + +try: + if host == "three.example.test": + for index in range(3): + fetch("/r%d" % index) + finish() + if host == "slow.example.test": + fetch("/slow") + fetch("/after-slow") + finish() + if host == "backoff.example.test": + fetch("/first") + time.sleep(1.5) + fetch("/second") + finish() + if host == "double.example.test": + guard.admit() + # A scanner that bypasses its own client and asks for a second grant on the wire. + guard.sock.sendall(b'{{"op":"admit","kind":"http"}}\n') + time.sleep(10) + if host == "hang.example.test": + guard.admit() + time.sleep(30) + if host == "nogate.example.test": + finish() +except AdmissionRefused as exc: + sys.stdout.write(json.dumps({{"ok": False, "error": "fixture_refused"}}) + "\n") + raise SystemExit(2) +''' + + +class _FlipGate: + """Supervisor-side gate double: admits a fixed budget and can be cancelled at any moment.""" + + def __init__(self, budget=100): + self.budget = budget + self.cancelled = threading.Event() + self.admits = 0 + self.probes = 0 + self.log = [] + + def admit(self): + self.log.append(("admit", time.monotonic())) + if self.cancelled.is_set(): + raise AdmissionDenied("job_cancelled") + if self.admits >= self.budget: + raise AdmissionDenied("budget_exhausted") + self.admits += 1 + return Grant(self.admits, 1.0, 10.0, self.budget - self.admits) + + def probe(self): + self.probes += 1 + if self.cancelled.is_set(): + raise AdmissionDenied("job_cancelled") + + +@unittest.skipUnless(sys.platform.startswith("linux"), "the supervisor uses Linux process isolation APIs") +class SupervisorAdmissionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = Path(self.temp.name) + integrations = root / "integrations" + worker = integrations / "worker" + worker.mkdir(parents=True) + (integrations / "__init__.py").write_text("", encoding="utf-8") + (worker / "__init__.py").write_text("", encoding="utf-8") + for name in ("admission.py", "network.py", "protocol.py"): + (worker / name).symlink_to(REPO_ROOT / "integrations" / "worker" / name) + self.root = root + self.worker = worker + patches = [ + mock.patch("integrations.worker.runner.os.geteuid", return_value=0), + mock.patch("integrations.worker.runner._has_kill_capability", return_value=True), + mock.patch("integrations.worker.runner._linux_preexec_limits", lambda: None), + mock.patch.object(admission, "PROBE_INTERVAL_SECONDS", 0.05), + ] + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + + def runner_for(self, recorder): + (self.worker / "scanner.py").write_text( + _CHILD_SCRIPT.format(port=recorder.port, result=json.dumps(_result())), encoding="utf-8" + ) + return SubprocessRunner(repo_root=str(self.root)) + + def test_a_scan_without_a_gate_never_starts(self): + with self.assertRaises(ScannerFailure) as refused: + SubprocessRunner(repo_root=str(self.root)).run("https://three.example.test/", timeout=5) + self.assertEqual(refused.exception.slug, "admission_guard_required") + + def test_every_request_is_admitted_and_the_scan_completes(self): + recorder = _Recorder(self) + gate = _FlipGate() + result = self.runner_for(recorder).run("https://three.example.test/", timeout=10, gate=gate) + self.assertEqual(result["status"], "no_evidence") + self.assertEqual(gate.admits, 3) + self.assertEqual([path for _t, path in recorder.requests], ["/r0", "/r1", "/r2"]) + + def test_budget_exhaustion_stops_the_scan_at_the_boundary(self): + recorder = _Recorder(self) + gate = _FlipGate(budget=2) + with self.assertRaises(ScannerFailure) as stopped: + self.runner_for(recorder).run("https://three.example.test/", timeout=10, gate=gate) + self.assertEqual(stopped.exception.slug, "admission_budget_exhausted") + self.assertEqual([path for _t, path in recorder.requests], ["/r0", "/r1"]) + + def test_cancellation_before_the_first_request_sends_nothing(self): + recorder = _Recorder(self) + gate = _FlipGate() + gate.cancelled.set() + with self.assertRaises(ScannerFailure) as stopped: + self.runner_for(recorder).run("https://three.example.test/", timeout=10, gate=gate) + self.assertEqual(stopped.exception.slug, "admission_job_cancelled") + self.assertEqual(recorder.requests, []) + + def test_cancellation_during_an_in_flight_request_aborts_it_and_admits_nothing_further(self): + recorder = _Recorder(self, delay=6.0) + gate = _FlipGate() + started = time.monotonic() + cancelled_at = [] + + def cancel_when_request_arrives(): + while not recorder.requests: + time.sleep(0.01) + cancelled_at.append(time.monotonic()) + gate.cancelled.set() + + thread = threading.Thread(target=cancel_when_request_arrives, daemon=True) + thread.start() + with self.assertRaises(ScannerFailure) as stopped: + self.runner_for(recorder).run("https://slow.example.test/", timeout=20, gate=gate) + thread.join(2) + self.assertEqual(stopped.exception.slug, "admission_job_cancelled") + self.assertEqual([path for _t, path in recorder.requests], ["/slow"], "only the already admitted request was sent") + self.assertEqual(gate.admits, 1) + self.assertLess(time.monotonic() - cancelled_at[0], 1.5, "the admitted request was not aborted promptly") + self.assertLess(time.monotonic() - started, 5.0, "the supervisor waited for the slow response") + + def test_cancellation_during_backoff_between_requests_admits_nothing(self): + recorder = _Recorder(self) + gate = _FlipGate() + + def cancel_after_first(): + while not recorder.requests: + time.sleep(0.01) + time.sleep(0.3) # the scanner is now idle in its backoff + gate.cancelled.set() + + thread = threading.Thread(target=cancel_after_first, daemon=True) + thread.start() + with self.assertRaises(ScannerFailure) as stopped: + self.runner_for(recorder).run("https://backoff.example.test/", timeout=10, gate=gate) + thread.join(2) + self.assertEqual(stopped.exception.slug, "admission_job_cancelled") + self.assertEqual([path for _t, path in recorder.requests], ["/first"]) + + def test_request_not_finished_within_the_disclosed_bound_is_killed(self): + recorder = _Recorder(self) + gate = _FlipGate() + short = Grant(1, 1.0, 0.3, 5) + gate.admit = lambda: (gate.log.append(("admit", time.monotonic())), short)[1] + with mock.patch.object(admission, "REQUEST_GRACE_SECONDS", 0.1): + with self.assertRaises(ScannerFailure) as stopped: + self.runner_for(recorder).run("https://hang.example.test/", timeout=10, gate=gate) + self.assertEqual(stopped.exception.slug, "request_deadline_exceeded") + + def test_a_scanner_that_breaks_the_protocol_is_killed(self): + recorder = _Recorder(self) + with self.assertRaises(ScannerFailure) as stopped: + self.runner_for(recorder).run("https://double.example.test/", timeout=10, gate=_FlipGate()) + self.assertEqual(stopped.exception.slug, "admission_protocol_violation") + + def test_a_result_is_never_accepted_once_admission_was_refused(self): + # The child finishes normally after a refusal it ignored; the supervisor still fails the run. + recorder = _Recorder(self) + gate = _FlipGate(budget=0) + with self.assertRaises(ScannerFailure): + self.runner_for(recorder).run("https://three.example.test/", timeout=10, gate=gate) + + +class ServiceWiringTests(unittest.TestCase): + def test_the_worker_service_hands_the_gate_to_the_runner_and_reports_the_failure(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + spool = DurableSpool(Path(temp.name) / "spool") + spool.prepare() + api = _FakeAPI(_claim()) + + class Runner: + gate = None + + def run(self, target, *, timeout, stop_event=None, gate=None): + Runner.gate = gate + raise ScannerFailure("admission_job_cancelled") + + service = WorkerService(api, spool, Runner(), REVISION, clock=lambda: NOW) + self.assertEqual(service.process_once(), "callback_accepted") + self.assertIsInstance(Runner.gate, AdmissionGate) + receipt = json.loads(api.result_bodies[0]) + self.assertEqual((receipt["outcome"], receipt["error"], receipt["result"]), ("failed", "admission_job_cancelled", None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_scan_worker.py b/test_scan_worker.py index a5cf0ff..e201ec0 100644 --- a/test_scan_worker.py +++ b/test_scan_worker.py @@ -114,13 +114,23 @@ def result(self, body): return self.result_response +class _UnusedGate: + """The isolation fixtures never contact a target, so their gate is never consulted.""" + + def admit(self): + raise AssertionError("fixture scanner unexpectedly asked for admission") + + def probe(self): + return None + + class _FakeRunner: def __init__(self, result=None): self.result = result or _result() self.calls = [] - def run(self, target, *, timeout, stop_event=None): - self.calls.append((target, timeout, stop_event)) + def run(self, target, *, timeout, stop_event=None, gate=None): + self.calls.append((target, timeout, stop_event, gate)) result = dict(self.result) result["input_url"] = target result["final_url"] = target @@ -581,7 +591,7 @@ def setUp(self): self.secret.write_bytes(b"supervisor-secret-material") self.secret.chmod(0o600) result_literal = repr(json.dumps(_result(), separators=(",", ":"))) - script = f'''import json, os, sys, time + script = f'''import json, os, socket, sys, time from urllib.parse import urlsplit def safe_excepthook(kind, _value, _traceback): sys.stdout.write(json.dumps({{"ok":False,"error":"fixture_" + kind.__name__.lower()}}) + "\\n") @@ -619,6 +629,14 @@ def safe_excepthook(kind, _value, _traceback): except BaseException as exc: sys.stdout.write(json.dumps({{"ok":False,"error":"fixture_" + type(exc).__name__.lower()}}) + "\\n") raise SystemExit(2) +if host == "admit.example.test": + channel = socket.socket(fileno=int(os.environ["CT_ADMISSION_FD"])) + channel.sendall(b'{{"op":"admit","kind":"http"}}\\n') + reply = b"" + while not reply.endswith(b"\\n"): + reply += channel.recv(64) + channel.sendall(b'{{"op":"done"}}\\n') + result["notes"] = ["child_uid=" + str(os.geteuid()), "admission_ok=" + str(json.loads(reply)["ok"]).lower()] sys.stdout.write(json.dumps({{"ok":True,"result":result}}, separators=(",", ":")) + "\\n") ''' scanner = worker / "scanner.py" @@ -630,27 +648,45 @@ def safe_excepthook(kind, _value, _traceback): def test_child_is_unprivileged_has_no_secret_env_or_key_access(self): with mock.patch.dict(os.environ, {"CT_SCAN_SECRET_FILE": str(self.secret)}): - result = self.runner.run("https://security.example.test/", timeout=5) + result = self.runner.run("https://security.example.test/", timeout=5, gate=_UnusedGate()) self.assertIn("child_uid=65534", result["notes"]) self.assertIn("secret_env_present=false", result["notes"]) self.assertIn("secret_file=denied", result["notes"]) + def test_unprivileged_child_can_ask_the_supervisor_for_admission_over_the_inherited_socket(self): + from integrations.worker.admission import Grant + + class Gate: + admits = 0 + + def admit(self): + Gate.admits += 1 + return Grant(1, 1.0, 10.0, 19) + + def probe(self): + return None + + result = self.runner.run("https://admit.example.test/", timeout=5, gate=Gate()) + self.assertIn("child_uid=65534", result["notes"]) + self.assertIn("admission_ok=true", result["notes"]) + self.assertEqual(Gate.admits, 1) + def test_parent_kills_child_at_deadline_and_caps_stdout(self): started = time.monotonic() with self.assertRaises(ScannerFailure) as timeout: - self.runner.run("https://timeout.example.test/", timeout=0.35) + self.runner.run("https://timeout.example.test/", timeout=0.35, gate=_UnusedGate()) self.assertEqual(timeout.exception.slug, "scanner_timeout") self.assertLess(time.monotonic() - started, 2.0) with self.assertRaises(ScannerFailure) as overflow: - self.runner.run("https://oversize.example.test/", timeout=5) + self.runner.run("https://oversize.example.test/", timeout=5, gate=_UnusedGate()) self.assertEqual(overflow.exception.slug, "scanner_output_overflow") def test_child_failure_and_partial_output_are_not_interpreted_as_results(self): with self.assertRaises(ScannerFailure) as failure: - self.runner.run("https://failure.example.test/", timeout=5) + self.runner.run("https://failure.example.test/", timeout=5, gate=_UnusedGate()) self.assertEqual(failure.exception.slug, "fixture_failure") with self.assertRaises(ScannerFailure) as partial: - self.runner.run("https://partial.example.test/", timeout=5) + self.runner.run("https://partial.example.test/", timeout=5, gate=_UnusedGate()) self.assertEqual(partial.exception.slug, "scanner_invalid_output") From a49467dc2ce6e8eba29f52911e41c0ff17a055cc Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Thu, 22 Oct 2026 06:54:44 +0000 Subject: [PATCH 2/7] Give the packaged-runtime smoke test a gate; skip Linux-only descriptor test elsewhere Co-Authored-By: Claude Code --- deployment/test-runtime-image.py | 17 ++++++++++++++++- test_admission.py | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/deployment/test-runtime-image.py b/deployment/test-runtime-image.py index ecd7b1d..0714567 100644 --- a/deployment/test-runtime-image.py +++ b/deployment/test-runtime-image.py @@ -5,6 +5,7 @@ import sys import tempfile +from integrations.worker.admission import AdmissionDenied from integrations.worker.runner import SubprocessRunner, _linux_preexec_limits from integrations.worker.service import load_secret @@ -30,7 +31,21 @@ env={'PATH': '/usr/local/bin:/usr/bin:/bin'}, preexec_fn=_linux_preexec_limits) -result = SubprocessRunner().run('https://runtime-fixture.example.invalid/', timeout=15) +class RefusingGate: + # With no network the lookup fails before any admission is requested; if the + # scanner ever asked, a refusal here would fail the run. + admits = 0 + + def admit(self): + RefusingGate.admits += 1 + raise AdmissionDenied('job_cancelled') + + def probe(self): + return None + + +result = SubprocessRunner().run('https://runtime-fixture.example.invalid/', timeout=15, gate=RefusingGate()) +assert RefusingGate.admits == 0, 'an admission was requested although the destination never resolved' assert result['status'] == 'error', result['status'] assert result['php2shell_status'] == 'unknown' assert result['php2shell_exploitability'] == 'not_tested' diff --git a/test_admission.py b/test_admission.py index 497c0d4..921fe51 100644 --- a/test_admission.py +++ b/test_admission.py @@ -577,6 +577,7 @@ def test_checker_sends_nothing_when_admission_is_refused(self): self.assertEqual(recorder.requests, []) +@unittest.skipUnless(sys.platform.startswith("linux"), "descriptor semantics differ outside Linux") class ScannerChannelTests(unittest.TestCase): def test_scanner_has_no_channel_without_a_valid_unix_socket(self): from integrations.worker import scanner From 0562f19dc3ddab1979c584257f8da77d9533b88d Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Thu, 22 Oct 2026 07:16:10 +0000 Subject: [PATCH 3/7] Harden the admission guard after independent review - Sequence-numbered requests and replies; any channel error poisons the scanner's channel, so a late grant can never answer a later request. - Refused admissions now end the run as failures (AdmissionRefused is no longer an OSError the checker swallows into a "completed" result). - Measure the one-second lifetime and ten-second bound from just before the call WordPress granted; bound every supervisor call to WordPress to four seconds; run liveness probes on their own thread so a slow store cannot delay the in-flight deadline or the kill. - A disabled integration (operator kill switch) reported by a probe now stops the scanner; only an unreachable guard is non-fatal for the request already admitted. - Admission no longer runs table-wide maintenance under the queue lock; expiry and lease are checked on the row. - A job already running at the schema 2 to 3 upgrade gets no fresh budget. - Tests: real scanner entry point, compare-and-set loss, injected database fault, operator-revocation lock wait, upgrade budget, desynchronisation. Co-Authored-By: Claude Code --- docs/WORDPRESS.md | 46 +++-- integrations/wordpress/ct-scan.php | 17 +- integrations/wordpress/tests/admission.php | 62 +++++- integrations/worker/admission.py | 191 +++++++++++++----- integrations/worker/scanner.py | 10 +- test_admission.py | 214 +++++++++++++++++++-- 6 files changed, 454 insertions(+), 86 deletions(-) diff --git a/docs/WORDPRESS.md b/docs/WORDPRESS.md index 4b15802..2546a89 100644 --- a/docs/WORDPRESS.md +++ b/docs/WORDPRESS.md @@ -107,26 +107,41 @@ Enforcement in the supervisor and scanner: - No grant, no connection: the network adapter asks for admission after destination validation and before opening any socket, re-checks that the grant is under one second old immediately before connecting, and reports - completion. A scanner started without the supervisor socket refuses to run. -- One grant is outstanding at a time. A second request, an unexpected message - or an oversized message ends the scan (`admission_protocol_violation`). -- The supervisor measures the disclosed ten-second request bound from the - grant (plus one second of grace) and kills the scanner on overrun + completion. A scanner started without the supervisor socket refuses to run, + and a refused admission ends the run as a failure (`admission_`), not + as an ordinary fetch error. +- Requests and replies on the socket carry a sequence number and any channel + error poisons the channel, so a late reply can never be read as the answer to + a later request. Every supervisor call to WordPress has a four-second wall + clock budget (a slower store is a refusal); one admission, including bounded + pacing and busy retries, takes at most about eight seconds. +- The grant's one-second lifetime and ten-second request bound are measured + from just before the call WordPress granted, not from delivery to the + scanner. A grant that aged out while the supervisor waited is not delivered. +- One grant is outstanding at a time. A second request, an out-of-sequence or + unexpected message or an oversized message ends the scan + (`admission_protocol_violation`). +- The supervisor kills the scanner when a request is not reported finished + within the ten-second bound plus one second of grace (`request_deadline_exceeded`). -- While the scanner runs, the supervisor probes the job about every two - seconds, including during backoff, and kills the scanner as soon as - WordPress reports that it is cancelled, expired or revoked. The abort is - therefore bounded by the probe interval plus the five-second probe timeout, - and by the ten-second request bound in any case. -- An unreachable guard refuses the next admission (`admission_guard_unavailable`) - and does not extend the bound on a request already admitted. Expired - authorization is refused locally without a call. +- While the scanner runs, the supervisor probes the job about every two seconds + on a separate thread (so a slow store cannot delay the deadline or the kill), + including during backoff, and kills the scanner as soon as WordPress reports + that the job is cancelled, expired or revoked, or that the integration is + disabled (the operator kill switch). Cancellation therefore aborts the one + admitted request within about two seconds plus the probe's own latency, and + never later than the ten-second bound. A probe that cannot reach WordPress + does not abort a request already admitted, and the next admission fails closed + (`admission_guard_unavailable`). Expired authorization is refused locally + without a call. - `SubprocessRunner.run` refuses to start without a gate. A refusal fails the run with an `admission_` category (for example `admission_job_cancelled`, `admission_budget_exhausted`). The result for a job already cancelled is rejected by WordPress as before. A budget or time stop is -a failed receipt, never a clean result, and no automatic rescan follows. +a failed receipt, never a clean result, and no automatic rescan follows. A job +that was already running when the schema is upgraded to version 3 has no record +of what it sent, so it receives no fresh budget (its admissions are refused). What this does not do: it cannot constrain a compromised scanner that ignores the protocol and opens its own sockets. The unprivileged UID and the @@ -137,6 +152,9 @@ deny private, link-local and metadata ranges) is the necessary second boundary. responsibility. DNS lookups of the already authorized hostname happen before admission and are not treated as target requests. The supervisor therefore also enforces the deadline and the stop by killing the scanner's process group. +Admission answers are protected by TLS to the fixed API origin but are not +themselves signed; a party able to forge that TLS session could mint grants, +which the egress boundary above still limits. Receipt retries stop at authorization expiry or one hour after lease expiry, whichever comes first. Permanent rejection keeps the receipt in the private diff --git a/integrations/wordpress/ct-scan.php b/integrations/wordpress/ct-scan.php index 8c61875..c946ab6 100644 --- a/integrations/wordpress/ct-scan.php +++ b/integrations/wordpress/ct-scan.php @@ -111,7 +111,8 @@ function ct_scan_nonces_table() { /** Create plugin tables once; MU plugins do not receive activation hooks. */ function ct_scan_maybe_install_schema() { - if ( get_option( 'ct_scan_schema_version' ) === CT_SCAN_SCHEMA_VERSION ) { + $previous = get_option( 'ct_scan_schema_version' ); + if ( $previous === CT_SCAN_SCHEMA_VERSION ) { return; } global $wpdb; @@ -184,6 +185,12 @@ function ct_scan_maybe_install_schema() { return; } } + if ( is_string( $previous ) && '' !== $previous && version_compare( $previous, '3', '<' ) ) { + // A job that started before request counting existed has no record of what it already + // sent: it gets no fresh budget, so its remaining admissions are refused. + $limits = ct_scan_profile_limits( ct_scan_current_profile() ); + $wpdb->query( $wpdb->prepare( "UPDATE {$jobs} SET http_admitted = %d WHERE status = 'running'", (int) $limits['http_requests'] ) ); + } update_option( 'ct_scan_schema_version', CT_SCAN_SCHEMA_VERSION, false ); } add_action( 'init', 'ct_scan_maybe_install_schema', 2 ); @@ -1033,11 +1040,17 @@ function ct_scan_admit_locked( $data, $now_ms ) { $refuse = static function ( $status, $reason, $extra = array() ) { return array( (int) $status, array_merge( array( 'ok' => false, 'reason' => sanitize_key( $reason ) ), $extra ) ); }; - ct_scan_maintenance_locked(); + // No table-wide maintenance here: probes run every couple of seconds under the same + // lock as Stop, so this path stays short. Expiry and lease are checked on the row. $row = ct_scan_row( $data['id'] ); if ( ! $row ) { return $refuse( 410, 'job_not_found' ); } + if ( 'running' === $row->status && strtotime( $row->authorization_expires_at . ' UTC' ) <= time() ) { + $wpdb->update( $table, array( 'status' => 'expired', 'error_code' => 'authorization_expired', 'finished_at' => ct_scan_utc_now(), 'lease_hash' => null, 'lease_until' => null, 'confirmation_token_hash' => null ), array( 'job_id' => (int) $row->job_id, 'status' => 'running' ), array( '%s', '%s', '%s', '%s', '%s', '%s' ), array( '%d', '%s' ) ); + ct_scan_mark_request_status( (int) $row->request_id, 'expired', 'authorization_expired' ); + return $refuse( 410, 'job_expired' ); + } if ( 'cancelled' === $row->status ) { return $refuse( 410, 'job_cancelled' ); } diff --git a/integrations/wordpress/tests/admission.php b/integrations/wordpress/tests/admission.php index d6e3098..ad63194 100644 --- a/integrations/wordpress/tests/admission.php +++ b/integrations/wordpress/tests/admission.php @@ -172,6 +172,18 @@ function ct_test_stop_token( $job_id ) { fclose( $pipes[2] ); proc_close( $holder ); +// Operator revocation waits for an in-progress admission like a stop does, and then blocks admission. +$job5b = ct_test_running_job( 'admit-revoke-wait.example.net' ); +$request_ids[] = $job5b['request_id']; +$holder = proc_open( array( PHP_BINARY, __DIR__ . '/lock-holder.php', '1.3' ), array( 1 => array( 'pipe', 'w' ), 2 => array( 'pipe', 'w' ) ), $pipes, null, null ); +ct_test_assert( is_resource( $holder ) && "locked\n" === fgets( $pipes[1] ), 'The lock-holder helper did not acquire the queue lock for revocation.' ); +$started = microtime( true ); +ct_test_assert( true === ct_scan_admin_revoke( $job5b['job_id'] ) && microtime( true ) - $started > 0.3, 'Operator revocation did not wait for the in-progress lock holder.' ); +ct_test_assert( 410 === ct_test_admit( $job5b )->get_status(), 'A job was admitted after operator revocation that waited on the lock.' ); +fclose( $pipes[1] ); +fclose( $pipes[2] ); +proc_close( $holder ); + // Operator revocation behaves like a stop for admission. $job5 = ct_test_running_job( 'admit-revoked.example.net' ); $request_ids[] = $job5['request_id']; @@ -192,9 +204,53 @@ function ct_test_stop_token( $job_id ) { $wpdb->update( $jobs, array( 'lease_until' => gmdate( 'Y-m-d H:i:s', time() - 2 ) ), array( 'job_id' => $job7['job_id'] ) ); ct_test_age_admission( $job7['job_id'] ); $lapsed = ct_test_admit( $job7 ); -ct_test_assert( 410 === $lapsed->get_status() && 1 === (int) ct_test_job_fields( $job7['job_id'] )->http_admitted && 'failed_unknown' === ct_test_job_fields( $job7['job_id'] )->status, 'A lapsed lease was admitted or its counters reset.' ); +ct_test_assert( 410 === $lapsed->get_status() && 'lease_invalid_or_expired' === $lapsed->get_data()['reason'] && 1 === (int) ct_test_job_fields( $job7['job_id'] )->http_admitted, 'A lapsed lease was admitted or its counters reset.' ); $reclaim = ct_test_rest_call( 'ct_scan_claim', '/ct-scan/v1/claim', array() ); -ct_test_assert( null === $reclaim->get_data()['job'] && 1 === (int) ct_test_job_fields( $job7['job_id'] )->http_admitted, 'Recovery produced a fresh claim or budget for a started job.' ); +ct_test_assert( null === $reclaim->get_data()['job'] && 1 === (int) ct_test_job_fields( $job7['job_id'] )->http_admitted && 'failed_unknown' === ct_test_job_fields( $job7['job_id'] )->status, 'Recovery produced a fresh claim or budget for a started job, or did not close it.' ); + +// Fault injection: a competing writer wins the reservation (the lock was lost) or the write throws. +// The compare-and-set must refuse rather than overwrite, and an exception must refuse rather than grant. +$job9 = ct_test_running_job( 'admit-fault.example.net' ); +$request_ids[] = $job9['request_id']; +$raced = false; +$competitor = static function ( $sql ) use ( &$raced, $jobs, $job9 ) { + global $wpdb; + if ( ! $raced && 0 === strpos( ltrim( $sql ), "UPDATE `{$jobs}` SET `http_admitted`" ) ) { + $raced = true; + $wpdb->query( $wpdb->prepare( "UPDATE {$jobs} SET http_admitted = 7 WHERE job_id = %d", $job9['job_id'] ) ); + } + return $sql; +}; +add_filter( 'query', $competitor ); +$lost = ct_test_admit( $job9 ); +remove_filter( 'query', $competitor ); +ct_test_assert( $raced && 503 === $lost->get_status() && 'admission_write_failed' === $lost->get_data()['reason'] && 7 === (int) ct_test_job_fields( $job9['job_id'] )->http_admitted, 'A lost reservation race was granted or overwritten.' ); +ct_test_age_admission( $job9['job_id'] ); +$wpdb->update( $jobs, array( 'http_admitted' => 0, 'last_admit_ms' => null ), array( 'job_id' => $job9['job_id'] ) ); +$thrower = static function ( $sql ) use ( $jobs ) { + if ( 0 === strpos( ltrim( $sql ), "UPDATE `{$jobs}` SET `http_admitted`" ) ) { + throw new RuntimeException( 'injected database fault' ); + } + return $sql; +}; +add_filter( 'query', $thrower ); +$faulted = ct_test_admit( $job9 ); +remove_filter( 'query', $thrower ); +ct_test_assert( 503 === $faulted->get_status() && 'admission_error' === $faulted->get_data()['reason'] && 0 === (int) ct_test_job_fields( $job9['job_id'] )->http_admitted, 'A database fault during admission did not fail closed.' ); +$wpdb->get_var( 'SELECT 1' ); +ct_test_assert( ct_scan_lock_acquire(), 'The queue lock was not released after a faulted admission.' ); +ct_scan_lock_release(); +ct_test_end_job( $job9['job_id'] ); + +// Schema upgrade: a job already running when request counting appears gets no fresh budget. +$job10 = ct_test_running_job( 'admit-upgrade.example.net' ); +$request_ids[] = $job10['request_id']; +update_option( 'ct_scan_schema_version', '2', false ); +ct_scan_maybe_install_schema(); +ct_test_assert( get_option( 'ct_scan_schema_version' ) === CT_SCAN_SCHEMA_VERSION && 20 === (int) ct_test_job_fields( $job10['job_id'] )->http_admitted, 'The v2 to v3 upgrade gave a running job a fresh request budget.' ); +ct_test_age_admission( $job10['job_id'] ); +ct_test_assert( 409 === ct_test_admit( $job10 )->get_status(), 'A running job from before the upgrade was still admitted.' ); +ct_test_end_job( $job10['job_id'] ); // Unavailable authorization store: a schema mismatch disables the whole worker API, including admission. $job8 = ct_test_running_job( 'admit-store.example.net' ); @@ -206,7 +262,7 @@ function ct_test_stop_token( $job_id ) { ct_test_end_job( $job8['job_id'] ); // Global operator kill switch: with the integration disabled, nothing is admitted. -$disabled = proc_open( array( PHP_BINARY, __DIR__ . '/admission-disabled.php' ), array( 1 => array( 'pipe', 'w' ), 2 => array( 'pipe', 'w' ) ), $dpipes, null, array_merge( $_ENV, array( 'CT_TEST_ENABLED' => '0', 'WORDPRESS_DB_PASSWORD' => 'fixture-only', 'WORDPRESS_DB_HOST' => 'db', 'CT_TEST_WP_ROOT' => getenv( 'CT_TEST_WP_ROOT' ) ?: '/var/www/html', 'PATH' => getenv( 'PATH' ) ) ) ); +$disabled = proc_open( array( PHP_BINARY, __DIR__ . '/admission-disabled.php' ), array( 1 => array( 'pipe', 'w' ), 2 => array( 'pipe', 'w' ) ), $dpipes, null, array_merge( getenv(), array( 'CT_TEST_ENABLED' => '0', 'WORDPRESS_DB_PASSWORD' => 'fixture-only', 'WORDPRESS_DB_HOST' => 'db', 'CT_TEST_WP_ROOT' => getenv( 'CT_TEST_WP_ROOT' ) ?: '/var/www/html', 'PATH' => getenv( 'PATH' ) ) ) ); $out = stream_get_contents( $dpipes[1] ); $err = stream_get_contents( $dpipes[2] ); proc_close( $disabled ); diff --git a/integrations/worker/admission.py b/integrations/worker/admission.py index 2209630..e520f01 100644 --- a/integrations/worker/admission.py +++ b/integrations/worker/admission.py @@ -11,6 +11,10 @@ violation or unavailable store is a refusal. * A grant is reserved and consumed in one step, is valid for at most ``MAX_VALID_SECONDS`` and covers exactly one connection and request. +* Every supervisor call to WordPress has a hard wall-clock budget; a call that + overruns it is a refusal. Requests and replies on the scanner socket carry a + sequence number, and any channel error poisons the channel, so a late reply + can never be read as the answer to a later request. * Only one grant may be outstanding. The supervisor kills the scanner if a request is not reported finished within ``MAX_REQUEST_SECONDS`` (plus a short grace), if the scanner skips the protocol, or if WordPress reports that the @@ -21,8 +25,9 @@ import json import re +import threading import time -from dataclasses import dataclass +from dataclasses import dataclass, replace from .protocol import APIError @@ -32,6 +37,7 @@ REQUEST_GRACE_SECONDS = 1.0 PROBE_INTERVAL_SECONDS = 2.0 MAX_ADMIT_WAIT_SECONDS = 4.0 +CALL_BUDGET_SECONDS = 4.0 BUSY_RETRY_SECONDS = 0.15 MAX_MESSAGE_BYTES = 256 CHILD_REPLY_TIMEOUT_SECONDS = 15.0 @@ -54,6 +60,9 @@ class Grant: valid_seconds: float timeout_seconds: float remaining: int + # Supervisor monotonic time just before the call that WordPress granted: the + # grant's one-second lifetime and ten-second bound are measured from here. + issued_at: float = 0.0 def _is_int(value) -> bool: @@ -81,6 +90,9 @@ def parse_grant(value) -> Grant: def parse_refusal(error: APIError) -> AdmissionDenied: """Turn a non-2xx answer into a refusal; unknown or malformed answers stay refusals.""" payload = getattr(error, "payload", None) + if isinstance(payload, dict) and payload.get("error") == "ct_scan_disabled": + # The operator kill switch (or an incomplete schema upgrade): not merely "unreachable". + return AdmissionDenied("integration_disabled") reason = payload.get("reason") if isinstance(payload, dict) else None if not isinstance(reason, str) or not REASON_RE.fullmatch(reason): reason = "guard_unavailable" @@ -114,21 +126,37 @@ def _check_local(self): raise AdmissionDenied("authorization_expired") def _call(self, kind: str): - try: - response = self.api.admit(self.claim, kind) - except APIError as exc: - if exc.status is None: - raise AdmissionDenied("guard_unavailable") from exc - raise parse_refusal(exc) from exc - return response + """One WordPress call under a hard wall-clock budget (a slow store is a refusal).""" + box = {} + + def run(): + try: + box["value"] = self.api.admit(self.claim, kind) + except BaseException as exc: # reported to the caller as a refusal below + box["error"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(CALL_BUDGET_SECONDS) + if worker.is_alive(): + raise AdmissionDenied("guard_timeout") + error = box.get("error") + if isinstance(error, APIError): + if error.status is None: + raise AdmissionDenied("guard_unavailable") from error + raise parse_refusal(error) from error + if error is not None: + raise AdmissionDenied("guard_unavailable") from error + return box["value"] def admit(self) -> Grant: """Reserve exactly one request, or raise AdmissionDenied. Retries are bounded.""" deadline = self._monotonic() + MAX_ADMIT_WAIT_SECONDS while True: self._check_local() + started = self._monotonic() try: - return parse_grant(self._call("http")) + return replace(parse_grant(self._call("http")), issued_at=started) except AdmissionDenied as denied: if self._monotonic() >= deadline: raise @@ -152,6 +180,11 @@ class AdmissionPump: ``service`` and ``tick`` return a fatal reason slug when the scanner must be killed, and ``None`` otherwise. After a fatal reason the pump answers nothing. + + Messages are single JSON lines: ``{"op":"admit","kind":"http","n":N}`` answered + by ``{"ok":true,"n":N,"valid_ms":M}`` or ``{"ok":false,"n":N,"reason":R}``, and + ``{"op":"done","n":N}``. ``N`` counts the scanner's admissions from 1 and must + advance by exactly one. """ def __init__(self, sock, gate, *, monotonic=time.monotonic): @@ -160,7 +193,11 @@ def __init__(self, sock, gate, *, monotonic=time.monotonic): self._monotonic = monotonic self._buffer = bytearray() self._inflight_until = None + self._inflight_n = None + self._last_n = 0 self._last_probe = monotonic() + self._probe_thread = None + self._probe_result = None self.grants = 0 self.fatal = None self.closed = False @@ -188,26 +225,37 @@ def _handle(self, line: bytes): message = json.loads(line.decode("ascii")) except (UnicodeDecodeError, json.JSONDecodeError): return self._fail("admission_protocol_violation") - if not isinstance(message, dict): + if not isinstance(message, dict) or not _is_int(message.get("n")): return self._fail("admission_protocol_violation") - if message == {"op": "done"}: - if not self.inflight: + number = message["n"] + if set(message) == {"op", "n"} and message["op"] == "done": + if not self.inflight or number != self._inflight_n: return self._fail("admission_protocol_violation") self._inflight_until = None + self._inflight_n = None return None - if message == {"op": "admit", "kind": "http"}: - if self.inflight: - # One connection at a time: a second grant is never available. + if set(message) == {"op", "kind", "n"} and message["op"] == "admit" and message["kind"] == "http": + if self.inflight or number != self._last_n + 1: + # One connection at a time and strictly sequential requests. return self._fail("admission_protocol_violation") + self._last_n = number try: grant = self.gate.admit() except AdmissionDenied as denied: - self._send({"ok": False, "reason": denied.slug}) + self._send({"ok": False, "n": number, "reason": denied.slug}) return self._fail(denied.slug) - # The deadline starts when WordPress granted, before the reply is delivered. - self._inflight_until = self._monotonic() + grant.timeout_seconds + REQUEST_GRACE_SECONDS + elapsed = self._monotonic() - grant.issued_at if grant.issued_at else 0.0 + valid = grant.valid_seconds - elapsed + if valid < 0.05: + # The grant outlived its lifetime while the supervisor waited on the guard. + self._send({"ok": False, "n": number, "reason": "admission_expired"}) + return self._fail("admission_expired") + # The disclosed bounds are measured from just before WordPress granted. + started = grant.issued_at or self._monotonic() + self._inflight_until = started + grant.timeout_seconds + REQUEST_GRACE_SECONDS + self._inflight_n = number self.grants += 1 - if not self._send({"ok": True, "valid_ms": int(grant.valid_seconds * 1000)}): + if not self._send({"ok": True, "n": number, "valid_ms": int(valid * 1000)}): return self._fail("admission_channel_failed") return None return self._fail("admission_protocol_violation") @@ -240,29 +288,55 @@ def service(self): return self._fail("admission_protocol_violation") return None + def _probe_worker(self): + try: + self.gate.probe() + self._probe_result = None + except AdmissionDenied as denied: + self._probe_result = denied + except BaseException: + self._probe_result = AdmissionDenied("guard_unavailable") + + def wait_probe(self, timeout: float = 10.0): + """Wait for an outstanding probe (tests, and orderly shutdown).""" + thread = self._probe_thread + if thread is not None: + thread.join(timeout) + def tick(self): - """Enforce the in-flight bound and watch for cancellation, expiry or revocation.""" + """Enforce the in-flight bound and watch for cancellation, expiry or revocation. + + Probes run on their own thread so a slow store can never delay the in-flight + deadline, a shutdown or the kill itself. + """ if self.fatal: return self.fatal now = self._monotonic() if self._inflight_until is not None and now > self._inflight_until: return self._fail("request_deadline_exceeded") - if now - self._last_probe >= PROBE_INTERVAL_SECONDS: + thread = self._probe_thread + if thread is not None and not thread.is_alive(): + self._probe_thread = None + denied, self._probe_result = self._probe_result, None + # An unreachable guard does not abort the request already admitted (its own + # ten-second bound applies), and the next admission fails closed. An explicit + # refusal, including a disabled integration, stops the scanner now. + if denied is not None and denied.slug not in {"guard_unavailable", "guard_timeout", "admission_response_invalid"}: + return self._fail(denied.slug) + thread = None + if self._probe_thread is None and now - self._last_probe >= PROBE_INTERVAL_SECONDS: self._last_probe = now - try: - self.gate.probe() - except AdmissionDenied as denied: - # An unreachable guard does not abort the request already admitted (its own - # ten-second bound applies), and the next admission fails closed. An explicit - # refusal stops the scanner now. - if denied.slug not in {"guard_unavailable", "admission_response_invalid"}: - return self._fail(denied.slug) + self._probe_result = None + self._probe_thread = threading.Thread(target=self._probe_worker, daemon=True) + self._probe_thread.start() return None -class AdmissionRefused(OSError): - """Raised inside the scanner when no admission was granted. Subclasses OSError so - that an unmodified checker treats it as a failed fetch and sends nothing.""" +class AdmissionRefused(Exception): + """Raised inside the scanner when no admission was granted. + + Deliberately not an OSError: the checker must not swallow it into an ordinary + fetch error, because a refused admission ends the run as a failure.""" class ChildAdmission: @@ -274,6 +348,18 @@ def __init__(self, sock, *, monotonic=time.monotonic): self._granted_at = None self._valid_seconds = 0.0 self._outstanding = False + self._n = 0 + self._dead = False + + def _poison(self, reason: str): + """After any channel error no later byte may be read as an answer.""" + self._dead = True + self._outstanding = False + try: + self.sock.close() + except OSError: + pass + raise AdmissionRefused(reason) def _read_line(self) -> bytes: deadline = self._monotonic() + CHILD_REPLY_TIMEOUT_SECONDS @@ -281,47 +367,52 @@ def _read_line(self) -> bytes: while True: remaining = deadline - self._monotonic() if remaining <= 0: - raise AdmissionRefused("admission_timeout") - self.sock.settimeout(remaining) + self._poison("admission_timeout") try: + self.sock.settimeout(remaining) chunk = self.sock.recv(1) - except OSError as exc: - raise AdmissionRefused("admission_channel_failed") from exc + except OSError: + self._poison("admission_channel_failed") if not chunk: - raise AdmissionRefused("admission_channel_closed") + self._poison("admission_channel_closed") if chunk == b"\n": return bytes(data) data.extend(chunk) if len(data) > MAX_MESSAGE_BYTES: - raise AdmissionRefused("admission_reply_invalid") + self._poison("admission_reply_invalid") def admit(self): + if self._dead: + raise AdmissionRefused("admission_channel_failed") if self._outstanding: raise AdmissionRefused("admission_outstanding") + self._n += 1 try: self.sock.settimeout(5.0) - self.sock.sendall(b'{"op":"admit","kind":"http"}\n') - except OSError as exc: - raise AdmissionRefused("admission_channel_failed") from exc + self.sock.sendall(json.dumps({"op": "admit", "kind": "http", "n": self._n}, separators=(",", ":")).encode("ascii") + b"\n") + except OSError: + self._poison("admission_channel_failed") try: reply = json.loads(self._read_line().decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AdmissionRefused("admission_reply_invalid") from exc - if not isinstance(reply, dict): - raise AdmissionRefused("admission_reply_invalid") + except (UnicodeDecodeError, json.JSONDecodeError): + self._poison("admission_reply_invalid") + if not isinstance(reply, dict) or reply.get("n") != self._n or not _is_int(reply.get("n")): + self._poison("admission_reply_invalid") if reply.get("ok") is not True: reason = reply.get("reason") + # A refusal is final for this scanner. + self._dead = True raise AdmissionRefused(reason if isinstance(reason, str) and REASON_RE.fullmatch(reason) else "admission_denied") valid_ms = reply.get("valid_ms") - if set(reply) != {"ok", "valid_ms"} or not _is_int(valid_ms) or not 1 <= valid_ms <= int(MAX_VALID_SECONDS * 1000): - raise AdmissionRefused("admission_reply_invalid") + if set(reply) != {"ok", "n", "valid_ms"} or not _is_int(valid_ms) or not 1 <= valid_ms <= int(MAX_VALID_SECONDS * 1000): + self._poison("admission_reply_invalid") self._granted_at = self._monotonic() self._valid_seconds = valid_ms / 1000.0 self._outstanding = True def ensure_fresh(self): """Call immediately before connecting: a stale grant is never used.""" - if not self._outstanding or self._granted_at is None: + if self._dead or not self._outstanding or self._granted_at is None: raise AdmissionRefused("admission_missing") if self._monotonic() - self._granted_at >= self._valid_seconds: raise AdmissionRefused("admission_expired") @@ -334,6 +425,6 @@ def done(self): self._granted_at = None try: self.sock.settimeout(2.0) - self.sock.sendall(b'{"op":"done"}\n') + self.sock.sendall(json.dumps({"op": "done", "n": self._n}, separators=(",", ":")).encode("ascii") + b"\n") except OSError: - pass + self._dead = True diff --git a/integrations/worker/scanner.py b/integrations/worker/scanner.py index 879e8ae..0271c93 100644 --- a/integrations/worker/scanner.py +++ b/integrations/worker/scanner.py @@ -12,7 +12,7 @@ from wordpress_check import Checker -from .admission import ChildAdmission +from .admission import AdmissionRefused, ChildAdmission from .network import PinnedOpener, RequestOrigin @@ -83,7 +83,13 @@ def main() -> int: guard = ChildAdmission(channel) checker = Checker(timeout=10.0, delay=1.0, contact=CONTACT) checker.opener = PinnedOpener(scope, guard=guard) - result = checker.check(target) + try: + result = checker.check(target) + except AdmissionRefused as exc: + # Not a fetch error: a refused admission ends the run as a failure. + reason = str(exc) + slug = reason if reason.startswith("admission_") else "admission_" + reason + return _write({"ok": False, "error": slug[:64]}, 2) return _write({"ok": True, "result": asdict(result)}, 0) except TimeoutError: return _write({"ok": False, "error": "scanner_timeout"}, 2) diff --git a/test_admission.py b/test_admission.py index 921fe51..d801f27 100644 --- a/test_admission.py +++ b/test_admission.py @@ -327,13 +327,28 @@ def test_admit_then_done_allows_the_next_admission(self): def test_a_second_admission_while_one_is_outstanding_kills_the_scanner(self): harness = _PumpHarness(self, _ScriptGate(admit=[_grant(), _grant()])) self.assertEqual(harness.child_admit()[0], {"ok": True}) - harness.child_sock.sendall(b'{"op":"admit","kind":"http"}\n') + harness.child_sock.sendall(b'{"op":"admit","kind":"http","n":2}\n') self.assertTrue(__import__("select").select([harness.parent], [], [], 1)[0]) self.assertEqual(harness.pump.service(), "admission_protocol_violation") self.assertEqual(harness.pump.grants, 1) def test_malformed_messages_are_protocol_violations(self): - for raw in (b"done\n", b'{"op":"done"}\n', b'{"op":"admit"}\n', b'{"op":"admit","kind":"tls"}\n', b'{"op":"admit","kind":"http","x":1}\n', b"[]\n", b"\xff\n", b"x" * 400 + b"\n", b"x" * 300): + for raw in ( + b"done\n", + b'{"op":"done"}\n', + b'{"op":"done","n":1}\n', + b'{"op":"admit","kind":"http"}\n', + b'{"op":"admit","kind":"tls","n":1}\n', + b'{"op":"admit","kind":"http","n":1,"x":1}\n', + b'{"op":"admit","kind":"http","n":2}\n', + b'{"op":"admit","kind":"http","n":0}\n', + b'{"op":"admit","kind":"http","n":true}\n', + b'{"op":"admit","kind":"http","n":"1"}\n', + b"[]\n", + b"\xff\n", + b"x" * 400 + b"\n", + b"x" * 300, + ): with self.subTest(raw=raw[:20]): harness = _PumpHarness(self, _ScriptGate(admit=[_grant()])) harness.child_sock.sendall(raw) @@ -360,16 +375,65 @@ def test_in_flight_deadline_is_enforced_by_the_supervisor(self): self.assertEqual(harness.pump.tick(), "request_deadline_exceeded") def test_probe_refusal_is_fatal_but_an_unreachable_guard_is_not_by_itself(self): + self.assertLessEqual(admission.PROBE_INTERVAL_SECONDS, 2.0, "cancellation must be noticed within a couple of seconds") clock = _Clock() gate = _ScriptGate(probe=[AdmissionDenied("guard_unavailable"), AdmissionDenied("job_cancelled")]) harness = _PumpHarness(self, gate, monotonic=clock) + self.assertIsNone(harness.pump.tick(), "no probe before the interval") + self.assertEqual(gate.probes, 0) clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 self.assertIsNone(harness.pump.tick()) + harness.pump.wait_probe() + self.assertIsNone(harness.pump.tick(), "an unreachable guard alone does not kill the admitted request") self.assertEqual(gate.probes, 1) - self.assertIsNone(harness.pump.tick(), "probes are paced") clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + harness.pump.tick() + harness.pump.wait_probe() self.assertEqual(harness.pump.tick(), "admission_job_cancelled") + def test_a_disabled_integration_is_a_refusal_that_stops_the_scanner(self): + clock = _Clock() + gate = _ScriptGate(probe=[AdmissionDenied("integration_disabled")]) + harness = _PumpHarness(self, gate, monotonic=clock) + clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + harness.pump.tick() + harness.pump.wait_probe() + self.assertEqual(harness.pump.tick(), "admission_integration_disabled") + down = parse_refusal(APIError("api_http_503", status=503, payload={"error": "ct_scan_disabled"})) + self.assertEqual(down.slug, "integration_disabled") + + def test_a_slow_probe_never_blocks_the_in_flight_deadline(self): + clock = _Clock() + release = threading.Event() + + class Slow(_ScriptGate): + def probe(self): + release.wait(5) + + harness = _PumpHarness(self, Slow(admit=[_grant(timeout=10.0)]), monotonic=clock) + self.assertEqual(harness.child_admit()[0], {"ok": True}) + clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + started = time.monotonic() + self.assertIsNone(harness.pump.tick()) + self.assertLess(time.monotonic() - started, 1.0, "tick() waited on the guard") + clock.now += 20 + self.assertEqual(harness.pump.tick(), "request_deadline_exceeded") + release.set() + harness.pump.wait_probe() + + def test_a_grant_that_aged_past_its_lifetime_while_the_supervisor_waited_is_not_delivered(self): + clock = _Clock() + + class Slow(_ScriptGate): + def admit(self): + clock.now += 1.5 # the guard call took longer than the grant lives + return Grant(1, 1.0, 10.0, 19, issued_at=1000.0) + + harness = _PumpHarness(self, Slow(), monotonic=clock) + outcome, fatal = harness.child_admit() + self.assertEqual((outcome, fatal), ({"refused": "admission_expired"}, "admission_expired")) + self.assertFalse(harness.pump.inflight) + def test_unreachable_guard_fails_the_next_admission_closed(self): class Down(_ScriptGate): def admit(self): @@ -380,6 +444,20 @@ def admit(self): outcome, fatal = harness.child_admit() self.assertEqual((outcome, fatal), ({"refused": "guard_unavailable"}, "admission_guard_unavailable")) + def test_a_guard_call_that_overruns_its_budget_is_a_refusal(self): + release = threading.Event() + + class Hang: + def admit(self, claim, kind): + release.wait(5) + + gate = AdmissionGate(Hang(), _claim(expires_at=NOW + 7200), clock=lambda: NOW) + with mock.patch.object(admission, "CALL_BUDGET_SECONDS", 0.1): + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + release.set() + self.assertEqual(denied.exception.slug, "guard_timeout") + def test_peer_close_is_remembered_without_spinning(self): harness = _PumpHarness(self, _ScriptGate()) harness.child_sock.close() @@ -395,13 +473,23 @@ def setUp(self): self.addCleanup(self.child.close) self.client = ChildAdmission(self.child, monotonic=self.clock) - def reply(self, value): - threading.Thread(target=lambda: (self.parent.recv(256), self.parent.sendall(value)), daemon=True).start() + def reply(self, value, *, number=None): + """Answer the next request. A dict gets the request's sequence number unless one is forced.""" + + def run(): + request = json.loads(self.parent.recv(256).decode()) + if isinstance(value, dict): + body = json.dumps(dict(value, n=request["n"] if number is None else number)).encode() + b"\n" + else: + body = value + self.parent.sendall(body) + + threading.Thread(target=run, daemon=True).start() def test_connect_is_refused_without_a_grant_and_after_the_grant_goes_stale(self): with self.assertRaises(AdmissionRefused): self.client.ensure_fresh() - self.reply(b'{"ok":true,"valid_ms":1000}\n') + self.reply({"ok": True, "valid_ms": 1000}) self.client.admit() self.clock.now += 0.5 self.client.ensure_fresh() @@ -411,25 +499,71 @@ def test_connect_is_refused_without_a_grant_and_after_the_grant_goes_stale(self) self.assertEqual(str(stale.exception), "admission_expired") def test_scanner_rejects_oversized_extended_or_garbled_replies(self): - for raw in (b'{"ok":true,"valid_ms":5000}\n', b'{"ok":true,"valid_ms":0}\n', b'{"ok":true,"valid_ms":1000,"x":1}\n', b'{"ok":true}\n', b'not json\n', b"[]\n", b"x" * 400 + b"\n"): - with self.subTest(raw=raw[:30]): + for raw in ( + {"ok": True, "valid_ms": 5000}, + {"ok": True, "valid_ms": 0}, + {"ok": True, "valid_ms": 1000, "x": 1}, + {"ok": True}, + b"not json\n", + b"[]\n", + b"x" * 400 + b"\n", + ): + with self.subTest(raw=raw if isinstance(raw, dict) else raw[:30]): + parent, child = socket.socketpair() + self.addCleanup(parent.close) + client = ChildAdmission(child, monotonic=self.clock) + self.parent, self.child = parent, child self.reply(raw) with self.assertRaises(AdmissionRefused): - self.client.admit() + client.admit() with self.assertRaises(AdmissionRefused): - self.client.ensure_fresh() + client.ensure_fresh() + + def test_a_reply_for_another_request_is_never_taken_as_the_answer(self): + # Desynchronisation: a stale grant arrives after the scanner gave up on its request. + self.reply({"ok": True, "valid_ms": 1000}, number=7) + with self.assertRaises(AdmissionRefused) as wrong: + self.client.admit() + self.assertEqual(str(wrong.exception), "admission_reply_invalid") + with self.assertRaises(AdmissionRefused): + self.client.admit() + with self.assertRaises(AdmissionRefused): + self.client.ensure_fresh() + + def test_a_late_reply_after_a_timeout_cannot_satisfy_the_next_request(self): + timeline = iter([1000.0, 1100.0]) + + def fake_monotonic(): + return next(timeline, 1100.0) - def test_refusal_reason_is_surfaced_and_a_closed_channel_refuses(self): - self.reply(b'{"ok":false,"reason":"job_cancelled"}\n') + client = ChildAdmission(self.child, monotonic=fake_monotonic) + with self.assertRaises(AdmissionRefused) as timeout: + client.admit() + self.assertEqual(str(timeout.exception), "admission_timeout") + try: + self.parent.sendall(b'{"ok":true,"n":1,"valid_ms":1000}\n') # the late answer + except OSError: + pass # the poisoned scanner end is already closed: the answer has nowhere to land + with self.assertRaises(AdmissionRefused): + client.admit() + with self.assertRaises(AdmissionRefused): + client.ensure_fresh() + + def test_refusal_reason_is_surfaced_and_is_final(self): + self.reply({"ok": False, "reason": "job_cancelled"}) with self.assertRaises(AdmissionRefused) as refused: self.client.admit() self.assertEqual(str(refused.exception), "job_cancelled") + with self.assertRaises(AdmissionRefused): + self.client.admit() + + def test_a_closed_channel_refuses(self): self.parent.close() with self.assertRaises(AdmissionRefused): self.client.admit() def test_a_second_admission_before_done_is_refused_in_the_scanner(self): - self.reply(b'{"ok":true,"valid_ms":1000}\n') + self.reply({"ok": True, "valid_ms": 1000}) self.client.admit() with self.assertRaises(AdmissionRefused): self.client.admit() @@ -572,8 +706,9 @@ def test_checker_sends_nothing_when_admission_is_refused(self): checker = Checker(timeout=2, delay=0) checker.opener = PinnedOpener(_LocalScope(recorder.port), guard=_Guard(allow=0)) with mock.patch.object(network, "_public_addresses", return_value=[(socket.AF_INET, "127.0.0.1")]): - result = checker.check("http://example.test/") - self.assertEqual(result.status, "error") + # Not swallowed into an ordinary fetch error: the refusal ends the run. + with self.assertRaises(AdmissionRefused): + checker.check("http://example.test/") self.assertEqual(recorder.requests, []) @@ -596,6 +731,55 @@ def test_scanner_has_no_channel_without_a_valid_unix_socket(self): self.assertIsNone(scanner._admission_channel(), "a TCP socket is not the supervisor channel") +@unittest.skipUnless(sys.platform.startswith("linux"), "the scanner entry point requires Linux") +class ScannerMainTests(unittest.TestCase): + """Drive the real scanner entry point, as the unprivileged child would run it.""" + + def run_main(self, env, target="https://example.test/"): + import io + + from integrations.worker import scanner + + stdout = io.BytesIO() + + class Out: + buffer = stdout + + class In: + buffer = io.BytesIO(json.dumps({"target": target}).encode()) + + with mock.patch.object(scanner.os, "geteuid", return_value=65534), mock.patch.object(scanner.sys, "stdin", In), mock.patch.object( + scanner.sys, "stdout", Out + ), mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + network, "_public_addresses", return_value=[(socket.AF_INET, "127.0.0.1")] + ): + code = scanner.main() + return code, json.loads(stdout.getvalue().decode()) + + def test_without_the_supervisor_channel_the_scanner_refuses_to_scan(self): + env = dict(os.environ) + env.pop("CT_ADMISSION_FD", None) + with mock.patch.dict(os.environ, env, clear=True): + code, output = self.run_main({}) + self.assertEqual((code, output), (2, {"ok": False, "error": "admission_channel_unavailable"})) + + def test_a_refused_admission_ends_the_run_as_a_failure_not_a_result(self): + parent, child = socket.socketpair() + self.addCleanup(parent.close) + fd = os.dup(child.fileno()) + child.close() + + def supervisor(): + request = json.loads(parent.recv(256).decode()) + parent.sendall(json.dumps({"ok": False, "n": request["n"], "reason": "job_cancelled"}).encode() + b"\n") + + thread = threading.Thread(target=supervisor, daemon=True) + thread.start() + code, output = self.run_main({"CT_ADMISSION_FD": str(fd)}) + thread.join(2) + self.assertEqual((code, output), (2, {"ok": False, "error": "admission_job_cancelled"})) + + # --- supervisor integration: a real child process, a local server, a flippable gate ----------- _CHILD_SCRIPT = r''' From 96b6b40d8a2d615485fa98ea959f7744ef9101e8 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Thu, 22 Oct 2026 07:17:41 +0000 Subject: [PATCH 4/7] Update the root-only isolation fixture to the sequence-numbered protocol Co-Authored-By: Claude Code --- test_scan_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_scan_worker.py b/test_scan_worker.py index e201ec0..3be7164 100644 --- a/test_scan_worker.py +++ b/test_scan_worker.py @@ -631,11 +631,11 @@ def safe_excepthook(kind, _value, _traceback): raise SystemExit(2) if host == "admit.example.test": channel = socket.socket(fileno=int(os.environ["CT_ADMISSION_FD"])) - channel.sendall(b'{{"op":"admit","kind":"http"}}\\n') + channel.sendall(b'{{"op":"admit","kind":"http","n":1}}\\n') reply = b"" while not reply.endswith(b"\\n"): reply += channel.recv(64) - channel.sendall(b'{{"op":"done"}}\\n') + channel.sendall(b'{{"op":"done","n":1}}\\n') result["notes"] = ["child_uid=" + str(os.geteuid()), "admission_ok=" + str(json.loads(reply)["ok"]).lower()] sys.stdout.write(json.dumps({{"ok":True,"result":result}}, separators=(",", ":")) + "\\n") ''' From d3d0a97c6ee465aa47e9a63960fefdb76fc6915d Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Thu, 22 Oct 2026 07:40:52 +0000 Subject: [PATCH 5/7] Re-ask for grants that came back too slowly; bound admission by the scan deadline Independent re-review of the admission guard found that a slow but successful WordPress round trip (about one second) made the delivered grant stale and the pump then killed the whole scan. The gate now discards a grant older than 0.6 s and asks again (bounded, slots stay spent), refuses after the scan deadline, lets shutdown interrupt a slow guard call, and the runner waits for an in-flight probe before releasing the API client. Co-Authored-By: Claude Code --- docs/WORDPRESS.md | 5 +++- integrations/worker/admission.py | 31 +++++++++++++++++---- integrations/worker/runner.py | 2 ++ integrations/worker/service.py | 2 +- test_admission.py | 46 ++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 7 deletions(-) diff --git a/docs/WORDPRESS.md b/docs/WORDPRESS.md index 2546a89..312c5f0 100644 --- a/docs/WORDPRESS.md +++ b/docs/WORDPRESS.md @@ -117,7 +117,10 @@ Enforcement in the supervisor and scanner: pacing and busy retries, takes at most about eight seconds. - The grant's one-second lifetime and ten-second request bound are measured from just before the call WordPress granted, not from delivery to the - scanner. A grant that aged out while the supervisor waited is not delivered. + scanner. A grant that took more than 0.6 s to come back is discarded and + asked for again (the discarded slot stays spent), at most twice; after that + the admission is refused (`admission_expired`). No admission is requested + after the scan's own deadline, and a shutdown interrupts a slow guard call. - One grant is outstanding at a time. A second request, an out-of-sequence or unexpected message or an oversized message ends the scan (`admission_protocol_violation`). diff --git a/integrations/worker/admission.py b/integrations/worker/admission.py index e520f01..e97c4e9 100644 --- a/integrations/worker/admission.py +++ b/integrations/worker/admission.py @@ -38,6 +38,10 @@ PROBE_INTERVAL_SECONDS = 2.0 MAX_ADMIT_WAIT_SECONDS = 4.0 CALL_BUDGET_SECONDS = 4.0 +# A grant that took longer than this to come back is discarded and asked for again (the +# discarded slot stays spent): the child must still have most of its one second. +MAX_GRANT_AGE_SECONDS = 0.6 +MAX_STALE_REGRANTS = 2 BUSY_RETRY_SECONDS = 0.15 MAX_MESSAGE_BYTES = 256 CHILD_REPLY_TIMEOUT_SECONDS = 15.0 @@ -105,10 +109,12 @@ def parse_refusal(error: APIError) -> AdmissionDenied: class AdmissionGate: """Supervisor-side client for the WordPress admission route.""" - def __init__(self, api, claim, *, stop_event=None, clock=time.time, monotonic=time.monotonic): + def __init__(self, api, claim, *, stop_event=None, deadline=None, clock=time.time, monotonic=time.monotonic): self.api = api self.claim = claim self.stop_event = stop_event + # Wall-clock (epoch) time after which no further admission is requested at all. + self.deadline = deadline self._clock = clock self._monotonic = monotonic @@ -124,6 +130,8 @@ def _check_local(self): raise AdmissionDenied("worker_shutdown") if self._clock() >= self.claim.expires_at.timestamp(): raise AdmissionDenied("authorization_expired") + if self.deadline is not None and self._clock() >= self.deadline: + raise AdmissionDenied("scan_deadline") def _call(self, kind: str): """One WordPress call under a hard wall-clock budget (a slow store is a refusal).""" @@ -137,9 +145,14 @@ def run(): worker = threading.Thread(target=run, daemon=True) worker.start() - worker.join(CALL_BUDGET_SECONDS) - if worker.is_alive(): - raise AdmissionDenied("guard_timeout") + budget_end = self._monotonic() + CALL_BUDGET_SECONDS + while worker.is_alive(): + # Sliced so shutdown interrupts a slow store; a late answer from an abandoned call is ignored. + worker.join(0.05) + if self.stop_event is not None and self.stop_event.is_set(): + raise AdmissionDenied("worker_shutdown") + if worker.is_alive() and self._monotonic() >= budget_end: + raise AdmissionDenied("guard_timeout") error = box.get("error") if isinstance(error, APIError): if error.status is None: @@ -152,11 +165,19 @@ def run(): def admit(self) -> Grant: """Reserve exactly one request, or raise AdmissionDenied. Retries are bounded.""" deadline = self._monotonic() + MAX_ADMIT_WAIT_SECONDS + stale = 0 while True: self._check_local() started = self._monotonic() try: - return replace(parse_grant(self._call("http")), issued_at=started) + grant = replace(parse_grant(self._call("http")), issued_at=started) + if self._monotonic() - started > MAX_GRANT_AGE_SECONDS: + # WordPress answered, but too slowly for the one-second lifetime to be useful. + stale += 1 + if stale > MAX_STALE_REGRANTS: + raise AdmissionDenied("admission_expired") + continue + return grant except AdmissionDenied as denied: if self._monotonic() >= deadline: raise diff --git a/integrations/worker/runner.py b/integrations/worker/runner.py index 3f0690f..8e463fc 100644 --- a/integrations/worker/runner.py +++ b/integrations/worker/runner.py @@ -269,6 +269,8 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None raise ScannerFailure(child_error or "scanner_process_failed") return self._parse_success(bytes(buffers["stdout"]), target, scope) finally: + # Let an in-flight liveness probe finish before the socket and the API client are reused. + pump.wait_probe(5.0) parent_sock.close() @staticmethod diff --git a/integrations/worker/service.py b/integrations/worker/service.py index e913105..ca9704f 100644 --- a/integrations/worker/service.py +++ b/integrations/worker/service.py @@ -257,7 +257,7 @@ def process_once(self): outcome, result, error = "failed", None, "stale_claim" else: try: - gate = AdmissionGate(self.api, claim, stop_event=self.stop_event, clock=self.clock) + gate = AdmissionGate(self.api, claim, stop_event=self.stop_event, deadline=self.clock() + scan_timeout, clock=self.clock) result = self.runner.run(claim.target, timeout=scan_timeout, stop_event=self.stop_event, gate=gate) SubprocessRunner.validate_result(result, claim.target, scope) outcome, error = "completed", "" diff --git a/test_admission.py b/test_admission.py index d801f27..2a2c27e 100644 --- a/test_admission.py +++ b/test_admission.py @@ -181,6 +181,52 @@ def test_endless_pacing_or_busy_answers_end_in_a_refusal(self): self.assertLess(len(self.api.calls), 100) self.assertLessEqual(self.clock.now - 1000.0, admission.MAX_ADMIT_WAIT_SECONDS + 1.1) + def test_a_grant_that_came_back_too_slowly_is_discarded_and_asked_for_again(self): + slow = [0.7, 0.7] + + class SlowAPI(_AdmitAPI): + def admit(inner, claim, kind): + if slow: + self.clock.now += slow.pop(0) + return super().admit(claim, kind) + + self.api = SlowAPI(GRANT, dict(GRANT, sequence=2), dict(GRANT, sequence=3)) + gate = AdmissionGate(self.api, _claim(expires_at=NOW + 7200), stop_event=self.stop, clock=lambda: NOW, monotonic=self.clock) + grant = gate.admit() + self.assertEqual(grant.sequence, 3, "the two slow slots are spent, the third is delivered") + self.assertEqual(self.api.calls, ["http", "http", "http"]) + # Persistently slow: refused rather than delivering a grant with no lifetime left. + slow.extend([0.7] * 5) + self.api = SlowAPI(*([GRANT] * 5)) + gate = AdmissionGate(self.api, _claim(expires_at=NOW + 7200), stop_event=self.stop, clock=lambda: NOW, monotonic=self.clock) + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + self.assertEqual(denied.exception.slug, "admission_expired") + self.assertEqual(len(self.api.calls), 3) + + def test_no_admission_is_requested_after_the_scan_deadline(self): + gate = AdmissionGate(_AdmitAPI(GRANT), _claim(expires_at=NOW + 7200), deadline=NOW - 1, clock=lambda: NOW, monotonic=self.clock) + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + self.assertEqual(denied.exception.slug, "scan_deadline") + + def test_shutdown_interrupts_a_slow_guard_call(self): + release = threading.Event() + + class Hang: + def admit(self_inner, claim, kind): + release.wait(5) + + stop = threading.Event() + gate = AdmissionGate(Hang(), _claim(expires_at=NOW + 7200), stop_event=stop, clock=lambda: NOW) + threading.Timer(0.2, stop.set).start() + started = time.monotonic() + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + release.set() + self.assertEqual(denied.exception.slug, "worker_shutdown") + self.assertLess(time.monotonic() - started, 2.0) + def test_expired_authorization_is_refused_locally_without_asking(self): gate = self.gate(GRANT, claim=_claim(expires_at=NOW - 1), wall=lambda: NOW) with self.assertRaises(AdmissionDenied) as denied: From 591bf4032d0793b9d4fa352619e04d674539ea75 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Thu, 22 Oct 2026 09:08:54 +0000 Subject: [PATCH 6/7] Admission guard: require stated egress enforcement; tighten bounds; review fixes - Config loader refuses to enable unless egress_enforced is literally true (the guard is advisory to a scanner that ignores it); docs, README, example config and an unverified nftables egress sketch state that activation requires enforced egress. - The in-flight bound is the disclosed ten seconds from the grant, with no extra grace. - Authorization is read from the database, not the object cache. - A completed result requires at least one admitted request. - A failed schema upgrade is logged once and not retried for five minutes; plugin version 0.2.0. - The destination check lists special-purpose ranges itself (CGNAT, documentation, benchmarking, 3fff::/20, 6to4, NAT64, ...) instead of relying on the interpreter's ipaddress properties. Co-Authored-By: Claude Code --- README.md | 4 +- deployment/egress-filter.nft.example | 47 ++++++++++++++++ deployment/test-http-fixture.php | 1 + deployment/wordpress-config.example.php | 4 ++ docs/WORDPRESS.md | 33 +++++++++-- integrations/wordpress/ct-scan.php | 21 ++++++- integrations/wordpress/tests/admission.php | 47 ++++++++++++++++ integrations/wordpress/tests/config-gate.php | 4 ++ integrations/wordpress/tests/integration.php | 4 ++ integrations/wordpress/tests/support.php | 1 + integrations/worker/admission.py | 3 +- integrations/worker/network.py | 59 ++++++++++++++++---- test_admission.py | 52 ++++++++++++++++- 13 files changed, 258 insertions(+), 22 deletions(-) create mode 100644 deployment/egress-filter.nft.example create mode 100644 integrations/wordpress/tests/config-gate.php diff --git a/README.md b/README.md index 44ef6f7..68c3b68 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,9 @@ shared development instructions; [CLAUDE.md](CLAUDE.md) points Claude to them. - [Full reference](docs/REFERENCE.md): every CLI option, output field, and request rule - [Assessment methodology](docs/PHP2SHELL.md): how each version is classified - [WordPress free-scan integration](docs/WORDPRESS.md): signed requests, confirmation, - private job queue, restricted worker, deployment and recovery + private job queue, restricted worker, deployment and recovery. **Activation + requires enforced network egress** (`egress_enforced`); the repository's + Compose file does not provide it. - [Contributing](CONTRIBUTING.md) ยท [Security policy](SECURITY.md) ```sh diff --git a/deployment/egress-filter.nft.example b/deployment/egress-filter.nft.example new file mode 100644 index 0000000..2dcc329 --- /dev/null +++ b/deployment/egress-filter.nft.example @@ -0,0 +1,47 @@ +# EXAMPLE ONLY. NOT VERIFIED in this repository's tests and not applied by deployment/compose.yaml. +# +# The per-request admission guard is advisory to a scanner that ignores it. Activation of the +# integration REQUIRES that outbound traffic from the worker container is restricted by something the +# scanner cannot change: this host-level nftables sketch shows the intended policy. Adapt it, test it +# from inside a throwaway container against a local listener (never a third-party host), and only +# then set `egress_enforced => true` in the protected plugin configuration. +# +# Intent for the worker container's bridge ("BRIDGE", set with the network option +# com.docker.network.bridge.name): +# - allow DNS to ONE resolver you control ("RESOLVER"), nothing else on port 53/853; +# - allow TCP 80 and 443 to PUBLIC addresses only (the customer's site, and the fixed API origin); +# - drop everything else, including every private, link-local, CGNAT, documentation, multicast and +# cloud-metadata range, and all IPv6 that is not global unicast. +table inet ct_php2shell_egress { + set deny_v4 { + type ipv4_addr + flags interval + elements = { + 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, + 192.0.0.0/24, 192.0.2.0/24, 192.88.99.0/24, 192.168.0.0/16, 198.18.0.0/15, + 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 240.0.0.0/4 + } + } + set deny_v6 { + type ipv6_addr + flags interval + elements = { + ::/128, ::1/128, ::ffff:0:0/96, 64:ff9b::/96, 64:ff9b:1::/48, 100::/64, 2001::/23, + 2001:db8::/32, 2002::/16, 3fff::/20, 5f00::/16, fc00::/7, fe80::/10, fec0::/10, ff00::/8 + } + } + chain forward { + type filter hook forward priority -10; policy accept; + iifname "BRIDGE" jump worker_egress + } + chain worker_egress { + ct state established,related accept + ip daddr RESOLVER udp dport 53 accept + ip daddr RESOLVER tcp dport 53 accept + ip daddr @deny_v4 counter drop + ip6 daddr @deny_v6 counter drop + ip6 daddr != 2000::/3 counter drop + tcp dport { 80, 443 } accept + counter drop + } +} diff --git a/deployment/test-http-fixture.php b/deployment/test-http-fixture.php index 238dd77..cf95ffb 100644 --- a/deployment/test-http-fixture.php +++ b/deployment/test-http-fixture.php @@ -10,6 +10,7 @@ file_put_contents($private . '/.htaccess', "Require all denied\n"); file_put_contents($private . '/config.php', ' true, + 'egress_enforced' => true, // disposable fixture: no network beyond the test network 'worker_hmac_secret' => str_repeat('fixture-', 8), 'expected_checker_revision' => str_repeat('a', 40), 'authorization_end_at' => gmdate('Y-m-d\TH:i:s\Z', time() + 86400 * 30), diff --git a/deployment/wordpress-config.example.php b/deployment/wordpress-config.example.php index bc7cf24..861e2fb 100644 --- a/deployment/wordpress-config.example.php +++ b/deployment/wordpress-config.example.php @@ -3,7 +3,11 @@ // The adjacent worker.key must contain the same random key as the worker volume. // Never put either deployed file in source control. return array( + // Keep false until EVERY item in docs/WORDPRESS.md "Egress" and the site's activation checklist is done. 'enabled' => false, + // Set true only after outbound traffic from the worker is restricted by something the scanner + // cannot change (see deployment/egress-filter.nft.example). While false the integration refuses to enable. + 'egress_enforced' => false, 'supported_terms_version' => '2026-10-21-r2', 'expected_checker_revision' => 'REPLACE_WITH_REVIEWED_40_CHARACTER_COMMIT_SHA', 'expected_rules_as_of' => '2026-10-21', diff --git a/docs/WORDPRESS.md b/docs/WORDPRESS.md index 312c5f0..bc6a4b1 100644 --- a/docs/WORDPRESS.md +++ b/docs/WORDPRESS.md @@ -125,7 +125,7 @@ Enforcement in the supervisor and scanner: unexpected message or an oversized message ends the scan (`admission_protocol_violation`). - The supervisor kills the scanner when a request is not reported finished - within the ten-second bound plus one second of grace + within ten seconds of the grant, with no extra grace (`request_deadline_exceeded`). - While the scanner runs, the supervisor probes the job about every two seconds on a separate thread (so a slow store cannot delay the deadline or the kill), @@ -170,6 +170,23 @@ Before upgrading the pinned revision or rotating the key, stop accepting new work, drain/reconcile existing receipts and stop the controller. Do not change the expected revision while old receipts are awaiting acknowledgement. +## Egress (activation requirement) + +The admission guard, the unprivileged UID and the resource limits are +cooperative controls: a scanner that is compromised and ignores the protocol can +open its own sockets, and the supervisor can only kill it after the fact. **The +integration therefore refuses to enable itself** (`ct_scan_enabled()` is false, +the whole worker API answers 503) unless the protected configuration contains +`'egress_enforced' => true`. Set that only after enforcing, outside the +container's control, that the worker may reach only: one DNS resolver you +control, TCP 80 and 443 to public addresses, and nothing in private, link-local, +CGNAT, documentation, multicast, cloud-metadata or non-global IPv6 ranges. +`deployment/compose.yaml` does **not** do this. `deployment/egress-filter.nft.example` +sketches the intended host policy; it is an unverified example, not a claim that +the repository enforces anything. The destination check inside the worker lists +those ranges itself (it does not depend on the Python version's `ipaddress` +properties), but that is a second layer, not the boundary. + ## Repository and site ownership The maintained integration is published in this repository. The CompleteTech @@ -292,10 +309,16 @@ The default configuration is disabled, and no historical request sweep exists. then start with the same command prefix and `up -d`. The API origin is fixed to `https://complete.tech`; changing sites requires a reviewed code change. Check for subnet conflicts before using the default worker subnet. -5. Apply the reviewed site patch/terms update, then set `enabled` true. The - runner must authenticate successfully before accepting live submissions. - A matching revision and key are required; mismatches fail closed. - +5. Enforce egress (see "Egress"), then set `egress_enforced` true. Apply the + reviewed site patch/terms update, then set `enabled` true. The runner must + authenticate successfully before accepting live submissions. A matching + revision and key and `egress_enforced` are required; otherwise the + integration stays disabled. + +Take a database backup first: the upgrade changes the jobs table in place. If it +cannot complete, the failure is written to the PHP error log once, the worker +API stays refused, and the upgrade is not retried for five minutes (so a broken +table cannot make every public request re-run it). Schema 3 adds `profile_id`, `http_admitted` and `last_admit_ms` to the jobs table (existing rows default to `r2-header-v1` with empty counters); the upgrade runs on `init` and the worker API refuses requests until it completes. diff --git a/integrations/wordpress/ct-scan.php b/integrations/wordpress/ct-scan.php index c946ab6..6ecf189 100644 --- a/integrations/wordpress/ct-scan.php +++ b/integrations/wordpress/ct-scan.php @@ -2,7 +2,7 @@ /** * Plugin Name: CompleteTech Authorized PHP2Shell Scan Queue * Description: Confirmation-gated, one-target PHP2Shell screening queue for Promo Requests. - * Version: 0.1.0 + * Version: 0.2.0 * * This MU plugin stores authorization and job state locally. It never starts a * scanner process. The external worker talks to the signed REST routes below. @@ -65,6 +65,10 @@ function ct_scan_config() { 'global_daily_limit' => 20, 'target_daily_limit' => 1, 'job_retention_days' => 365, + // The admission guard is advisory to a scanner that ignores it; only enforced network egress + // (allow the customer's HTTP/HTTPS, deny private/metadata ranges) makes it a boundary. The + // operator must state, deliberately, that this exists. See docs/WORDPRESS.md "Egress". + 'egress_enforced' => false, ); $path = defined( 'CT_SCAN_PRIVATE_CONFIG' ) @@ -86,6 +90,7 @@ function ct_scan_config() { $config['enabled'] = true === $config['enabled'] && is_string( $config['worker_hmac_secret'] ) && strlen( $config['worker_hmac_secret'] ) >= 32 + && true === $config['egress_enforced'] && is_string( $config['expected_checker_revision'] ) && 1 === preg_match( '/\A[a-f0-9]{40}\z/i', $config['expected_checker_revision'] ) && is_string( $config['supported_terms_version'] ) @@ -115,6 +120,10 @@ function ct_scan_maybe_install_schema() { if ( $previous === CT_SCAN_SCHEMA_VERSION ) { return; } + // A failed upgrade must not re-run dbDelta on every public request: back off for five minutes. + if ( get_transient( 'ct_scan_schema_backoff' ) ) { + return; + } global $wpdb; require_once ABSPATH . 'wp-admin/includes/upgrade.php'; $charset = $wpdb->get_charset_collate(); @@ -182,9 +191,13 @@ function ct_scan_maybe_install_schema() { foreach ( $required as $table => $columns ) { $found = $wpdb->get_col( "SHOW COLUMNS FROM {$table}", 0 ); if ( ! is_array( $found ) || array_diff( $columns, $found ) ) { + // Loud once per back-off window, and bounded. The worker API stays refused (fail closed). + error_log( 'ct-scan: database upgrade to schema ' . CT_SCAN_SCHEMA_VERSION . ' is incomplete for ' . $table . '; the worker API stays refused. Retrying in 5 minutes. Restore from backup or repair the table.' ); // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log + set_transient( 'ct_scan_schema_backoff', '1', 5 * MINUTE_IN_SECONDS ); return; } } + delete_transient( 'ct_scan_schema_backoff' ); if ( is_string( $previous ) && '' !== $previous && version_compare( $previous, '3', '<' ) ) { // A job that started before request counting existed has no record of what it already // sent: it gets no fresh budget, so its remaining admissions are refused. @@ -277,6 +290,8 @@ function ct_scan_meta_string( $post_id, $key ) { /** Check the saved request and acceptance snapshot before any job is created. */ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_terms = true, $frozen_expiry = '' ) { + // Authorization is judged on what the database holds now, never on an object-cache copy. + clean_post_cache( (int) $request_id ); $config = ct_scan_config(); $post = get_post( $request_id ); if ( ! $post || 'ct_promo_request' !== $post->post_type || 'private' !== $post->post_status ) { @@ -1300,6 +1315,10 @@ function ct_scan_result( WP_REST_Request $request ) { return ct_scan_rest_error( 410, 'authorization_revoked_or_expired' ); } if ( 'completed' === $data['outcome'] ) { + // A completed result is only credible if the guard admitted at least one request for this job. + if ( (int) $row->http_admitted < 1 ) { + return ct_scan_rest_error( 409, 'no_admission' ); + } $encoded_result = ct_scan_validate_result( $data['result'], $row ); if ( is_wp_error( $encoded_result ) ) { return ct_scan_rest_error( 422, $encoded_result->get_error_code() ); diff --git a/integrations/wordpress/tests/admission.php b/integrations/wordpress/tests/admission.php index ad63194..a997c87 100644 --- a/integrations/wordpress/tests/admission.php +++ b/integrations/wordpress/tests/admission.php @@ -268,6 +268,53 @@ function ct_test_stop_token( $job_id ) { proc_close( $disabled ); ct_test_assert( "disabled\n" === $out, 'The disabled-integration check failed: ' . $out . $err ); + +// Egress gate: without an explicit statement that network egress is enforced, the integration refuses to enable. +function ct_test_gate( $env ) { + $proc = proc_open( array( PHP_BINARY, __DIR__ . '/config-gate.php' ), array( 1 => array( 'pipe', 'w' ), 2 => array( 'pipe', 'w' ) ), $pipes, null, array_merge( getenv(), array( 'CT_TEST_WP_ROOT' => getenv( 'CT_TEST_WP_ROOT' ) ?: '/var/www/html' ), $env ) ); + $out = stream_get_contents( $pipes[1] ); + stream_get_contents( $pipes[2] ); + proc_close( $proc ); + return trim( $out ); +} +ct_test_assert( 'enabled' === ct_test_gate( array() ) && 'refused' === ct_test_gate( array( 'CT_TEST_EGRESS' => '0' ) ), 'The integration enabled itself without egress_enforced, or refused with it.' ); + +// Authorization is judged on the database, not on an object-cache copy: a change that bypassed the +// cache (a restore, a direct query, another process) ends a running job at once. +$jobc = ct_test_running_job( 'admit-cache.example.net' ); +$request_ids[] = $jobc['request_id']; +get_post_meta( $jobc['request_id'], '_ct_terms_sha256', true ); // prime the cache +get_post( $jobc['request_id'] ); +$wpdb->update( $wpdb->postmeta, array( 'meta_value' => str_repeat( '8', 64 ) ), array( 'post_id' => $jobc['request_id'], 'meta_key' => '_ct_terms_sha256' ) ); +ct_test_assert( str_repeat( '8', 64 ) !== get_post_meta( $jobc['request_id'], '_ct_terms_sha256', true ), 'Fixture expected a stale object-cache value.' ); +$stale = ct_test_admit( $jobc ); +ct_test_assert( 410 === $stale->get_status() && 'authorization_revoked_or_expired' === $stale->get_data()['reason'], 'A change made behind the object cache did not end the authorization.' ); + +// Schema upgrade failure is loud once and bounded: no dbDelta on every public request. +$log = tempnam( sys_get_temp_dir(), 'ctlog' ); +$previous_log = ini_set( 'error_log', $log ); +$wpdb->query( 'ALTER TABLE ' . $jobs . ' DROP COLUMN last_admit_ms' ); +update_option( 'ct_scan_schema_version', '2', false ); +delete_transient( 'ct_scan_schema_backoff' ); +$runs = 0; +$count = static function ( $queries ) use ( &$runs ) { + ++$runs; + return array(); // dbDelta does nothing: the upgrade stays incomplete +}; +add_filter( 'dbdelta_queries', $count ); +ct_scan_maybe_install_schema(); +ct_scan_maybe_install_schema(); +ct_scan_maybe_install_schema(); +remove_filter( 'dbdelta_queries', $count ); +$logged = (string) file_get_contents( $log ); +ct_test_assert( 2 === $runs, "dbDelta ran $runs times (two tables, once) for three requests after a failed upgrade." ); +ct_test_assert( 1 === substr_count( $logged, 'ct-scan: database upgrade' ) && false !== get_transient( 'ct_scan_schema_backoff' ), 'A failed upgrade was not logged exactly once with a back-off.' ); +ct_test_assert( 503 === ct_test_admit( $jobc )->get_status() && '2' === get_option( 'ct_scan_schema_version' ), 'The worker API was not refused during an incomplete upgrade.' ); +delete_transient( 'ct_scan_schema_backoff' ); +ct_scan_maybe_install_schema(); +ct_test_assert( get_option( 'ct_scan_schema_version' ) === CT_SCAN_SCHEMA_VERSION && false === get_transient( 'ct_scan_schema_backoff' ), 'The upgrade did not complete after the back-off.' ); +ini_set( 'error_log', (string) $previous_log ); +@unlink( $log ); // Remove only records created by this test. foreach ( $request_ids as $id ) { $wpdb->delete( $jobs, array( 'request_id' => $id ), array( '%d' ) ); diff --git a/integrations/wordpress/tests/config-gate.php b/integrations/wordpress/tests/config-gate.php new file mode 100644 index 0000000..78aeaa3 --- /dev/null +++ b/integrations/wordpress/tests/config-gate.php @@ -0,0 +1,4 @@ + $claimed['id'], 'lease' => $claimed['lease'], 'scope_hash' => $claimed['scope_hash'], 'outcome' => 'completed', 'result' => ct_test_result( $fields['target'] ), 'error' => '', 'checker_revision' => str_repeat( 'a', 40 ), ); +// A completed result is not credible unless the guard admitted at least one request for the job. +$unadmitted = ct_test_rest_call( 'ct_scan_result', '/ct-scan/v1/result', $run ); +ct_test_assert( 409 === $unadmitted->get_status() && 'no_admission' === $unadmitted->get_data()['error'] && 'running' === ct_scan_row( $claimed['id'] )->status, 'A completed result was accepted for a job that never had a request admitted.' ); +ct_test_assert( 200 === ct_test_rest_call( 'ct_scan_admit', '/ct-scan/v1/admit', array( 'id' => $claimed['id'], 'lease' => $claimed['lease'], 'scope_hash' => $claimed['scope_hash'], 'kind' => 'http' ) )->get_status(), 'Fixture admission was refused.' ); $result = ct_test_rest_call( 'ct_scan_result', '/ct-scan/v1/result', $run ); ct_test_assert( 200 === $result->get_status() && true === $result->get_data()['ok'], 'Valid checker receipt was not acknowledged.' ); $retry = ct_test_rest_call( 'ct_scan_result', '/ct-scan/v1/result', $run ); diff --git a/integrations/wordpress/tests/support.php b/integrations/wordpress/tests/support.php index 3db9e85..b1a39e5 100644 --- a/integrations/wordpress/tests/support.php +++ b/integrations/wordpress/tests/support.php @@ -10,6 +10,7 @@ $private_config = tempnam( '/tmp', 'ct-scan-integration-' ); file_put_contents( $private_config, ' '0' !== getenv( 'CT_TEST_ENABLED' ), + 'egress_enforced' => '0' !== getenv( 'CT_TEST_EGRESS' ), 'supported_terms_version' => '2026-10-21-r2', 'expected_checker_revision' => str_repeat( 'a', 40 ), 'expected_rules_as_of' => '2026-10-21', diff --git a/integrations/worker/admission.py b/integrations/worker/admission.py index e97c4e9..855f42a 100644 --- a/integrations/worker/admission.py +++ b/integrations/worker/admission.py @@ -34,7 +34,8 @@ MAX_VALID_SECONDS = 1.0 MAX_REQUEST_SECONDS = 10.0 -REQUEST_GRACE_SECONDS = 1.0 +# No grace: a request must be finished within the disclosed ten seconds of the grant. +REQUEST_GRACE_SECONDS = 0.0 PROBE_INTERVAL_SECONDS = 2.0 MAX_ADMIT_WAIT_SECONDS = 4.0 CALL_BUDGET_SECONDS = 4.0 diff --git a/integrations/worker/network.py b/integrations/worker/network.py index 7ebe0b8..4461ce6 100644 --- a/integrations/worker/network.py +++ b/integrations/worker/network.py @@ -131,6 +131,51 @@ def validate_request(self, value: str, *, mark_upgrade: bool = True) -> tuple[st return scheme, port, host, target +# Explicit special-purpose ranges, listed rather than inferred: Python's ipaddress `is_global` +# differs between versions (3.10 through 3.14 disagree on several of these), and a destination +# check must not depend on the interpreter. Anything in these ranges is never a customer's site. +DENIED_V4 = tuple( + ipaddress.ip_network(value) + for value in ( + "0.0.0.0/8", # "this" network + "10.0.0.0/8", + "100.64.0.0/10", # carrier-grade NAT + "127.0.0.0/8", + "169.254.0.0/16", # link-local, cloud metadata + "172.16.0.0/12", + "192.0.0.0/24", # IETF protocol assignments + "192.0.2.0/24", # documentation + "192.88.99.0/24", # deprecated 6to4 relay anycast + "192.168.0.0/16", + "198.18.0.0/15", # benchmarking + "198.51.100.0/24", # documentation + "203.0.113.0/24", # documentation + "224.0.0.0/4", # multicast + "240.0.0.0/4", # reserved, includes the broadcast address + ) +) +DENIED_V6 = tuple( + ipaddress.ip_network(value) + for value in ( + "::/128", + "::1/128", + "::ffff:0:0/96", # IPv4-mapped + "64:ff9b::/96", # NAT64 + "64:ff9b:1::/48", # local-use NAT64 + "100::/64", # discard-only + "2001::/23", # IETF protocol assignments, includes Teredo + "2001:db8::/32", # documentation + "2002::/16", # 6to4 + "3fff::/20", # documentation (RFC 9637) + "5f00::/16", # SRv6 SIDs + "fc00::/7", # unique local + "fe80::/10", # link-local + "fec0::/10", # deprecated site-local + "ff00::/8", # multicast + ) +) + + def _public_addresses(host: str, port: int, resolver=socket.getaddrinfo): """Resolve afresh and reject the entire answer set if any address is non-public.""" try: @@ -154,16 +199,6 @@ def _public_addresses(host: str, port: int, resolver=socket.getaddrinfo): address = ipaddress.ip_address(text) except (IndexError, ValueError) as exc: raise ValueError("invalid_dns_answer") from exc - denied_v4 = ( - ipaddress.ip_network("192.0.0.0/24"), - ipaddress.ip_network("192.88.99.0/24"), - ) - denied_v6 = ( - ipaddress.ip_network("64:ff9b::/96"), - ipaddress.ip_network("64:ff9b:1::/48"), - ipaddress.ip_network("2001::/23"), - ipaddress.ip_network("2002::/16"), - ) unsafe = ( not address.is_global or address.is_multicast @@ -174,11 +209,11 @@ def _public_addresses(host: str, port: int, resolver=socket.getaddrinfo): or address.is_private ) if isinstance(address, ipaddress.IPv4Address): - unsafe = unsafe or any(address in network for network in denied_v4) + unsafe = unsafe or any(address in network for network in DENIED_V4) if isinstance(address, ipaddress.IPv6Address): unsafe = unsafe or address.ipv4_mapped is not None or address.is_site_local unsafe = unsafe or address not in ipaddress.ip_network("2000::/3") - unsafe = unsafe or any(address in network for network in denied_v6) + unsafe = unsafe or any(address in network for network in DENIED_V6) if unsafe: raise ValueError("non_public_address") key = (family, address.compressed) diff --git a/test_admission.py b/test_admission.py index 2a2c27e..6abd985 100644 --- a/test_admission.py +++ b/test_admission.py @@ -10,6 +10,7 @@ import hashlib import http.server +import ipaddress import json import os import socket @@ -415,10 +416,11 @@ def test_in_flight_deadline_is_enforced_by_the_supervisor(self): gate = _ScriptGate(admit=[_grant(timeout=10.0)]) harness = _PumpHarness(self, gate, monotonic=clock) self.assertEqual(harness.child_admit()[0], {"ok": True}) - clock.now += 10.9 + clock.now += 9.9 self.assertIsNone(harness.pump.tick()) - clock.now += 0.2 + clock.now += 0.2 # 10.1 s after the grant: the disclosed ten-second bound, with no extra grace self.assertEqual(harness.pump.tick(), "request_deadline_exceeded") + self.assertEqual(admission.REQUEST_GRACE_SECONDS, 0.0) def test_probe_refusal_is_fatal_but_an_unreachable_guard_is_not_by_itself(self): self.assertLessEqual(admission.PROBE_INTERVAL_SECONDS, 2.0, "cancellation must be noticed within a couple of seconds") @@ -511,6 +513,52 @@ def test_peer_close_is_remembered_without_spinning(self): self.assertTrue(harness.pump.closed) +class NetworkDenylistTests(unittest.TestCase): + """The destination check lists special-purpose ranges itself; it must not depend on the + interpreter's ipaddress `is_*` properties, which differ between Python versions.""" + + @staticmethod + def _answer(text): + family = socket.AF_INET6 if ":" in text else socket.AF_INET + return (family, socket.SOCK_STREAM, 6, "", (text, 443)) + + def _rejected(self, text): + with self.assertRaises(ValueError): + network._public_addresses("example.test", 443, resolver=lambda *_a, **_k: [self._answer(text)]) + + def test_every_listed_range_is_refused_at_both_ends_and_in_the_middle(self): + for net in network.DENIED_V4 + network.DENIED_V6: + first, last = net[0], net[-1] + middle = net[net.num_addresses // 2] if net.num_addresses > 2 else first + for address in {first, last, middle}: + with self.subTest(network=str(net), address=str(address)): + self._rejected(str(address)) + + def test_the_list_alone_refuses_them_even_if_every_builtin_property_said_public(self): + neutral = property(lambda self: False) + patches = [] + for cls in (ipaddress.IPv4Address, ipaddress.IPv6Address): + for name in ("is_private", "is_reserved", "is_multicast", "is_unspecified", "is_loopback", "is_link_local"): + patches.append(mock.patch.object(cls, name, new=neutral)) + patches.append(mock.patch.object(cls, "is_global", new=property(lambda self: True))) + patches.append(mock.patch.object(ipaddress.IPv6Address, "is_site_local", new=neutral)) + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + for text in ("100.64.0.1", "192.0.2.1", "198.18.0.1", "198.51.100.1", "203.0.113.1", "10.1.2.3", "169.254.169.254", "0.0.0.1", "255.255.255.255", + "3fff::1", "3fff:fff:ffff:ffff::1", "2001:db8::1", "2002:808:808::", "64:ff9b::808:808", "100::1", "fc00::1", "fe80::1", "5f00::1"): + with self.subTest(address=text): + self._rejected(text) + + def test_neighbours_just_outside_each_range_are_not_swept_up(self): + for text in ("100.63.255.255", "100.128.0.0", "198.17.255.255", "198.20.0.0", "192.0.3.0", "203.0.114.0", "3fff:1000::1", "2001:4860:4860::8888", "2606:4700:4700::1111"): + with self.subTest(address=text): + self.assertEqual( + network._public_addresses("example.test", 443, resolver=lambda *_a, **_k: [self._answer(text)]), + [(socket.AF_INET6 if ":" in text else socket.AF_INET, text)], + ) + + class ChildAdmissionTests(unittest.TestCase): def setUp(self): self.clock = _Clock() From 0bd4510aefee621c32d3cd84ee24dc8656963c29 Mon Sep 17 00:00:00 2001 From: Timothy Gregg Date: Fri, 23 Oct 2026 15:31:47 -0400 Subject: [PATCH 7/7] Treat a busy or failing probe as inconclusive; bound stale re-asks by the admit deadline A liveness probe that met the no-wait queue lock (queue_busy), often held by the supervisor's own concurrent admission, or an internal admission_error, killed the scanner and spent the customer's daily scan. The probe now retries a busy lock briefly within one call budget and the pump treats queue_busy and admission_error like an unreachable guard: the admitted request keeps its ten-second bound and the next admission still fails closed. Explicit refusals remain fatal. The stale-grant re-ask path now checks the four-second admission deadline, so one admission stays within about eight seconds as documented. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/WORDPRESS.md | 15 ++++++--- integrations/worker/admission.py | 36 +++++++++++++++----- test_admission.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 13 deletions(-) diff --git a/docs/WORDPRESS.md b/docs/WORDPRESS.md index bc6a4b1..e3325ba 100644 --- a/docs/WORDPRESS.md +++ b/docs/WORDPRESS.md @@ -118,8 +118,9 @@ Enforcement in the supervisor and scanner: - The grant's one-second lifetime and ten-second request bound are measured from just before the call WordPress granted, not from delivery to the scanner. A grant that took more than 0.6 s to come back is discarded and - asked for again (the discarded slot stays spent), at most twice; after that - the admission is refused (`admission_expired`). No admission is requested + asked for again (the discarded slot stays spent), at most twice and never + after the admission's four-second retry deadline; after that the admission is + refused (`admission_expired`). No admission is requested after the scan's own deadline, and a shutdown interrupts a slow guard call. - One grant is outstanding at a time. A second request, an out-of-sequence or unexpected message or an oversized message ends the scan @@ -133,9 +134,13 @@ Enforcement in the supervisor and scanner: that the job is cancelled, expired or revoked, or that the integration is disabled (the operator kill switch). Cancellation therefore aborts the one admitted request within about two seconds plus the probe's own latency, and - never later than the ten-second bound. A probe that cannot reach WordPress - does not abort a request already admitted, and the next admission fails closed - (`admission_guard_unavailable`). Expired authorization is refused locally + never later than the ten-second bound. A probe that finds the queue lock held + elsewhere (often by the supervisor's own concurrent admission) retries briefly + within one call budget. An inconclusive probe (WordPress unreachable or slow, + a malformed answer, a still-busy lock or an internal `admission_error`) does + not abort a request already admitted or end the scan, and the next admission + fails closed on the same condition (for example + `admission_guard_unavailable`). Expired authorization is refused locally without a call. - `SubprocessRunner.run` refuses to start without a gate. diff --git a/integrations/worker/admission.py b/integrations/worker/admission.py index 855f42a..fedd217 100644 --- a/integrations/worker/admission.py +++ b/integrations/worker/admission.py @@ -48,6 +48,12 @@ CHILD_REPLY_TIMEOUT_SECONDS = 15.0 REASON_RE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") RETRYABLE_BUSY = {"queue_busy", "admission_write_failed"} +# Probe answers that say nothing about whether the job is live. They never abort a +# request already admitted (its own ten-second bound still applies) and the next +# admission fails closed on them. ``queue_busy`` only means the queue lock was held +# elsewhere (often by this supervisor's own concurrent admission); ``admission_error`` +# means WordPress failed while deciding. Every explicit refusal stays fatal. +PROBE_INCONCLUSIVE = {"guard_unavailable", "guard_timeout", "admission_response_invalid", "queue_busy", "admission_error"} class AdmissionDenied(Exception): @@ -175,7 +181,8 @@ def admit(self) -> Grant: if self._monotonic() - started > MAX_GRANT_AGE_SECONDS: # WordPress answered, but too slowly for the one-second lifetime to be useful. stale += 1 - if stale > MAX_STALE_REGRANTS: + if stale > MAX_STALE_REGRANTS or self._monotonic() >= deadline: + # Bounded by the same admission deadline as every other retry. raise AdmissionDenied("admission_expired") continue return grant @@ -190,9 +197,21 @@ def admit(self) -> Grant: raise def probe(self): - """Ask whether the job is still live. Consumes no budget and does not pace.""" - self._check_local() - response = self._call("probe") + """Ask whether the job is still live. Consumes no budget and does not pace. + + A busy queue lock is retried briefly within one call budget; if it stays + busy the probe reports ``queue_busy``, which the pump treats as inconclusive. + """ + busy_until = self._monotonic() + CALL_BUDGET_SECONDS + while True: + self._check_local() + try: + response = self._call("probe") + break + except AdmissionDenied as denied: + if denied.slug != "queue_busy" or self._monotonic() + BUSY_RETRY_SECONDS >= busy_until: + raise + self._wait(BUSY_RETRY_SECONDS) if not isinstance(response, dict) or set(response) != {"ok", "live"} or response["ok"] is not True or response["live"] is not True: raise AdmissionDenied("admission_response_invalid") @@ -340,10 +359,11 @@ def tick(self): if thread is not None and not thread.is_alive(): self._probe_thread = None denied, self._probe_result = self._probe_result, None - # An unreachable guard does not abort the request already admitted (its own - # ten-second bound applies), and the next admission fails closed. An explicit - # refusal, including a disabled integration, stops the scanner now. - if denied is not None and denied.slug not in {"guard_unavailable", "guard_timeout", "admission_response_invalid"}: + # An inconclusive probe (unreachable guard, busy queue lock, store error) does + # not abort the request already admitted (its own ten-second bound applies), + # and the next admission fails closed. An explicit refusal, including a + # disabled integration, stops the scanner now. + if denied is not None and denied.slug not in PROBE_INCONCLUSIVE: return self._fail(denied.slug) thread = None if self._probe_thread is None and now - self._last_probe >= PROBE_INTERVAL_SECONDS: diff --git a/test_admission.py b/test_admission.py index 6abd985..fd07ad6 100644 --- a/test_admission.py +++ b/test_admission.py @@ -205,6 +205,21 @@ def admit(inner, claim, kind): self.assertEqual(denied.exception.slug, "admission_expired") self.assertEqual(len(self.api.calls), 3) + def test_slow_regrants_stop_at_the_admission_deadline(self): + class SlowAPI(_AdmitAPI): + def admit(inner, claim, kind): + self.clock.now += 2.5 + return super().admit(claim, kind) + + self.api = SlowAPI(*([GRANT] * 5)) + gate = AdmissionGate(self.api, _claim(expires_at=NOW + 7200), stop_event=self.stop, clock=lambda: NOW, monotonic=self.clock) + with self.assertRaises(AdmissionDenied) as denied: + gate.admit() + self.assertEqual(denied.exception.slug, "admission_expired") + # The second slow answer lands past the four-second deadline: no third ask. + self.assertEqual(len(self.api.calls), 2) + self.assertLessEqual(self.clock.now - 1000.0, admission.MAX_ADMIT_WAIT_SECONDS + admission.CALL_BUDGET_SECONDS) + def test_no_admission_is_requested_after_the_scan_deadline(self): gate = AdmissionGate(_AdmitAPI(GRANT), _claim(expires_at=NOW + 7200), deadline=NOW - 1, clock=lambda: NOW, monotonic=self.clock) with self.assertRaises(AdmissionDenied) as denied: @@ -254,6 +269,23 @@ def test_probe_requires_an_exact_live_answer(self): self.assertEqual(denied.exception.slug, "job_cancelled") + def test_probe_retries_a_busy_queue_lock_within_one_call_budget(self): + self.gate(_refusal(503, "queue_busy"), _refusal(503, "queue_busy"), {"ok": True, "live": True}).probe() + self.assertEqual(self.api.calls, ["probe", "probe", "probe"]) + self.clock.now = 1000.0 + gate = self.gate(*([_refusal(503, "queue_busy")] * 100)) + with self.assertRaises(AdmissionDenied) as denied: + gate.probe() + self.assertEqual(denied.exception.slug, "queue_busy") + self.assertLess(len(self.api.calls), 100) + self.assertLessEqual(self.clock.now - 1000.0, admission.CALL_BUDGET_SECONDS) + # Only a busy lock is retried: an explicit refusal is reported at once. + with self.assertRaises(AdmissionDenied) as denied: + self.gate(_refusal(410, "authorization_revoked_or_expired"), {"ok": True, "live": True}).probe() + self.assertEqual(denied.exception.slug, "authorization_revoked_or_expired") + self.assertEqual(self.api.calls, ["probe"]) + + class ApiClientAdmitTests(unittest.TestCase): def test_admit_is_signed_for_its_own_route_with_a_short_timeout_and_exact_body(self): captured = {} @@ -439,6 +471,31 @@ def test_probe_refusal_is_fatal_but_an_unreachable_guard_is_not_by_itself(self): harness.pump.wait_probe() self.assertEqual(harness.pump.tick(), "admission_job_cancelled") + def test_a_busy_or_failing_store_during_a_probe_does_not_kill_the_scan(self): + # The supervisor's own admission can hold the no-wait queue lock while a probe runs. + clock = _Clock() + gate = _ScriptGate( + admit=[_grant(), _grant()], + probe=[AdmissionDenied("queue_busy"), AdmissionDenied("admission_error"), AdmissionDenied("authorization_revoked_or_expired")], + ) + harness = _PumpHarness(self, gate, monotonic=clock) + self.assertEqual(harness.child_admit()[0], {"ok": True}) + for _ in range(2): + clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + self.assertIsNone(harness.pump.tick()) + harness.pump.wait_probe() + self.assertIsNone(harness.pump.tick(), "an inconclusive probe does not end the scan") + harness.child.done() + __import__("select").select([harness.parent], [], [], 1) + self.assertIsNone(harness.pump.service()) + self.assertEqual(harness.child_admit(), ({"ok": True}, None), "the scan continues to its next admission") + self.assertEqual(gate.probes, 2) + # An explicit refusal from a later probe is still fatal. + clock.now += admission.PROBE_INTERVAL_SECONDS + 0.01 + harness.pump.tick() + harness.pump.wait_probe() + self.assertEqual(harness.pump.tick(), "admission_authorization_revoked_or_expired") + def test_a_disabled_integration_is_a_refusal_that_stops_the_scanner(self): clock = _Clock() gate = _ScriptGate(probe=[AdmissionDenied("integration_disabled")])