diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50be488f5f..21903a63ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1010,9 +1010,16 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_place.py + tests/integration/test_cell_update_directory.py + tests/integration/test_cell_update_addresses.py + tests/integration/test_cell_update_run_pg.py + tests/integration/test_cell_update_participants_ch.py + tests/integration/test_cell_place_cleanup.py tests/integration/test_cell_attach_two_replicas.py tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_scim_census_live_ch.py + tests/integration/test_cell_update_scim_live_ch.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py tests/integration/test_directory_control_cleanup.py @@ -1062,6 +1069,34 @@ jobs: bash dev/ci_start_postgres.sh docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby pytest -q tests/integration/test_pg_online_index_durability.py + - name: pytest placement durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-place-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_CELL_PLACE_DRILL: "1" + LOOM_CELL_PLACE_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-place-drill-pg loom-scale-place-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-place-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-place-drill-pg loom-scale-place-drill-pg-standby || true + docker network rm loom-scale-place-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-place-drill-pg loom-scale-place-drill-pg-standby + pytest -q tests/integration/test_cell_place_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 1360777087..a78e60bce4 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -493,8 +493,12 @@ replaces a later cell's under the directory lock after the same check and a fres echo. The first cell's probe and upstream addresses are copied from the router's configuration at registration, and `loom cell update` replaces them from it, once every router replica runs the new values, after the same probe and a check that the new -upstream reports the cell's deployment id. A prepare rerun never moves an address: it -refuses while the router replica it reaches is configured with others. +upstream reports the cell's deployment id. The generation/address CAS does not prove +all-router agreement: an old router can read fresh expected values and reverse the +addresses. Keep old endpoints reachable until every router refreshes its routing table +or restarts after the directory write; config rollout alone does not flush the cache, +which can persist indefinitely during directory failure. A prepare rerun never moves +an address: it refuses while the router replica it reaches is configured with others. **The `first` row.** Prepare writes an explicit, immutable `first` row into every cell's Postgres (`loom.cell_first`: a row trigger refuses its update and delete and a statement @@ -601,27 +605,43 @@ directory, both values of `loom.cell_self`. A prepared cell's issuers and org pa are reserved nowhere: the router routes no issuer to it and takes no placement from it until attach has run its overlap refusals and written its directory row. -A change to what a cell accepts goes through `loom cell update --config `, under -the same directory lock as attach; it computes `next` from that configuration with the -same function the processes use, and refuses while a different `next` is pending; rerun -with the same configuration, or `--finish`, first writes `next` wherever it is missing. -It reruns attach's overlap refusals against every other cell's directory row, counting -both values of each, writes the new fingerprint as `next` to the directory and then to -`loom.cell_self`, so old and new processes both serve while the new configuration rolls -out. Finalization first narrows the cell's accepted value durably while the directory -keeps both reservations. It then requires a complete post-cutoff participant census, +A change to what a cell accepts goes through `loom cell update --config ` +([operator procedure](../operations/cell-update.md)). The nonsecret settings file +overlays the CLI's effective settings without installing them. It computes `next` +with the same function the processes use and refuses while a different `next` is +pending. Under the directory lock, overlap checks count both values of every other +cell. Configured placements, their counts, issuer routing and the directory pair +commit together before local tenancy repair and acceptance of the pair. + +`--finish` selects next; `--revert` selects current. Each repairs local tenancy from +actual same-cell immutable placement winners for both reserved values, using an +operator read fenced by the generation, exact registration and pair, locked +root-secret fingerprint, audit and synchronous WAL acknowledgement. First-cell +lookup fallback is not evidence of a placement. The attaching first cell may still +lack placements for its preexisting configured orgs; other missing placements +require directory metadata recovery. + +Finalization durably narrows local acceptance while the directory keeps both +reservations. It then takes a complete post-cutoff participant census and requires permanent guarded retirement or verified exact death for every captured identity, -and a serving target replacement outside that captured set. Heartbeat expiry and -request deadlines do not prove this. Only afterward may the directory release its -old reservation. Online finalization needs the [lifecycle baseline](#participant-lifecycle-baseline) -and controller fencing; SCIM-on changes remain gated on actual per-cell SCIM support. A widened issuer or -pattern is reserved in the directory before any process accepts it, and a narrowed one -is released only after none does. A refusal or a wait that runs out keeps both values -and prints the processes not yet reporting `next`; rerunning resumes, and -`loom cell update --revert` waits the same way until no live process matches only `next` -(refusing when `--wait` runs out), then drops `next`, in `loom.cell_self` first, under -the same lock; a configured org the update placed stays placed in the cell (a placement -is never deleted). +including target processes, and a serving target replacement outside that set. +Only then may the directory release its old reservation. Heartbeat expiry, +request deadlines, missing participant rows and workers-only replacements do not +prove this. Online finalization needs the +[lifecycle baseline](#participant-lifecycle-baseline) and controller fencing. +For a SCIM-on target, the CLI runs the shared fresh provisioning census before +claiming the generation, after durable acceptance, and after participant turnover +before reservation release. It uses the target cell's configured SCIM credentials, +OIDC login and required authentication. `--revoke-unprovisioned` permits the same +reported, irreversible SCIM cutoffs as attach; without it, an unqualified live +subject refuses. Revert to SCIM-off current requires no provisioning census. + +A timeout after cutoff keeps local acceptance single and the directory dual; the +command reports incomplete turnover. Repeating the selected action takes a fresh +census. An identical `--config` repairs directory-first interruption and can widen +local acceptance again; subsequent finalization takes another cutoff and census. +A singleton no-op still proves local synchronous durability. Every configured org +placed by an update stays placed and retains its tenancy row after revert. A new SCIM token changes no fingerprint and needs no `loom cell update`. A cell changes its identity provider application as a single deployment does ([SSO and @@ -881,13 +901,26 @@ read-only scope, lists any tenancy row the directory does not place. it, unless the cell is the first and its mode reads `recording`, and an org-free command never uses it; a leaked one is stopped by `loom cell limit --closed`, which needs none of the cell's stores, and replaced by `loom cell credential` (below). - `loom cell place --org ` (declared org-free: it writes the org's placement and - tenancy row, which is how an org comes to be held), with the router's operator credential, + `loom cell place --cell --org ` (declared org-free: it writes the org's + placement and tenancy row, which is how an org comes to be held), with the router's operator credential, places an org for a command that writes a new one (a bot, an import), through the router's placement API and in its order, as attach's placement of configured orgs does: it inserts the placement and reads back the winner, writes the cell's tenancy row only if the winner is this cell (otherwise it refuses and writes nothing), and reports success only once the - row is written, so a rerun is idempotent. For an org the directory does not place yet it + row commits synchronously. The CLI holds the attach lock on a direct marked Postgres + session, requires a full enforced local cell state and the full accepted configuration + fingerprint, and checks both again before inserting. A transactional durability message + makes even a no-op insert flush an earlier primary-only tenancy row to the standby. + Opening the session, initial checks and the local transaction through COMMIT each have + a separate client deadline derived from `LOOM_PG_STATEMENT_TIMEOUT_MS`; the directory + request keeps its own transport timeout. Each local phase reserves cleanup inside its + original deadline. On failure the shared control owner closes the exact backend before + cancelling or reaping its work, deferring repeated signals within that same deadline. + Successful exit and failures outside a local phase have a separate cleanup allowance + derived from the same setting. An unknown outcome names rerunning the same command; + the permanent directory winner and idempotent local insert let that rerun finish. The + route checks directory state; a direct operator API caller is trusted to check local + Postgres mode as the CLI does. For an org the directory does not place yet it passes the placement API's gate (below): refused while the directory is unavailable, for a cell the gate does not accept, closed, or when the fingerprint of the CLI's own configuration of the cell (with the deployment id read from the stores it connects to) @@ -986,9 +1019,12 @@ read-only scope, lists any tenancy row the directory does not place. safe, since the read-back makes it idempotent. For an org the directory already places, a cell's call is checked only for its credential (against the cells' hashes the router caches for `LOOM_ROUTER_DIRECTORY_CACHE_S`, default 5 s) and the cell's - rate, and a `loom cell place` only for the operator credential; both answer the winner - whatever the cell's state, quota or fingerprint, adding no placement (the winner is - public through the lookup anyway); a request refused past its cell's rate answers 503 + rate. An operator's `loom cell place` checks its deployment id and fingerprint against + the requested cell's locked row even for an org already placed there; an org placed + elsewhere answers that winner and the CLI refuses without inserting local tenancy. + Existing placements bypass closed and quota checks. Every successful call writes an + audit row before its synchronous commit, including a retry of an existing winner, so + the returned winner survives failover. A request refused past its cell's rate answers 503 with `Retry-After` (at most a minute), and its retry is admitted by the replica that answers it once the cell's calls there are under the rate. An org the cell already holds needs no call, so serving it never depends on the router. @@ -1418,6 +1454,12 @@ that the same backend still holds ATTACH. A COMMIT, rollback or ownership-check never becomes a busy retry. After attempting the mode update, the diagnostic stays unknown until rollback plus ownership or COMMIT is acknowledged. +Placement and cell control bind their cleanup owner synchronously inside the shared +Postgres connector, before deployment or codec initialization can wait. A closed owner +refuses initialization and closes a late session within its original deadline. Process +admission, participant attribution and retirement seals still apply. Callers without an +explicit owner retain connector cleanup until initialization finishes. + Directory control mutations supervise the complete callback transaction independently, including acquisition and COMMIT. One 131-second bound includes public close, child settlement and pool-context exit, with cleanup time reserved inside it. Both a failed @@ -1433,29 +1475,37 @@ operation or cleanup failure. The shared directory pool opens physical sessions only for foreground acquisitions; pool object creation does not establish connectivity. Healthy released sessions are cached under the existing size cap, recycling and idle expiry. A later cold acquisition -opens a new session. Fixed JSON codec initialization quarantines its never-borrowed raw -session on failure; native interruptions return to that foreground consumer after driver -cleanup, with no autonomous opening or replenishment task. If quarantine fails, the -original failure records that session closure is unknown. +opens a new session. On fixed JSON codec initialization failure, an uncancelled +acquisition awaits the driver's disconnect acknowledgment before locally aborting the +never-borrowed raw session and running driver cleanup. An interrupted acquisition starts +no new close allowance; an unacknowledged close records that closure is unknown. Native +interruptions return to the foreground consumer after driver cleanup, with no autonomous +opening or replenishment task. The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded entry points and the named exceptions: -| Writer | Fence | -| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | -| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | -| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | -| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | -| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | -| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | -| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | - -Read-only routes are exempt. Router migrations and an offline directory restore are -maintenance operations. Future restore/update acceptance writers must use the same -claim and lock-session helpers; a new directory mutation must use the common comparison -before its no-op branch as well. Direct privileged SQL is not an online control writer. +| Writer | Fence | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | +| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | +| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | +| `cell_update_pg.write.change` | Fresh directory claim; owner's bounded transaction on the exact ATTACH backend, paired registration and exact observed local identity/pair before tenancy repair and acceptance write | +| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | +| `directory.update_addresses` | Claimed generation, full registration and exact pair under directory/row locks; commit-time address check, audit and synchronous WAL acknowledgement in one bounded transaction | +| `directory.update_configuration`, `_configuration_commit` | Claimed generation, full registration and exact pair under ordered org/directory/row locks; root-secret and reservation checks, placements, audit and acceptance commit in one bounded synchronous transaction | +| `directory.configuration_placements` | Claimed generation, full registration and exact pair under directory/row locks; root-secret and actual winner checks with audit and synchronous WAL acknowledgement in one bounded transaction | +| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | +| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | +| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | + +Ordinary read-only routes are exempt; `configuration_placements` is an audited, +WAL-fenced proof before local tenancy repair. Router migrations and an offline +directory restore are maintenance operations. Update acceptance writers use the +same claim and lock-session helpers; any additional acceptance writer must do so +as well. A directory mutation uses the common comparison before its no-op branch. +Direct privileged SQL is not an online control writer. The [control upgrade and recovery procedure](../operations/cell-control.md) defines the required fleet turnover and offline restore of generations. N-1 data-plane builds can read and insert on the additive schema; old control clients and router handlers cannot @@ -1974,10 +2024,12 @@ An integration test with two cells and a router in CI's Postgres job: value, a winner of this cell's name under another deployment id answers 503 and writes no tenancy row, and a process whose configuration differs from its cell's row places nothing; -- a cell attached with SCIM off and switched on by `loom cell update` refuses while an - agent-signup token is live, one minted while the update rolls out included, and a - process configured with another cell's issuer answers 503 to every bearer request, - reached through the router or directly; +- `loom cell update --config` and `--finish` freshly qualify a SCIM-on target before + any generation claim or acceptance write and recheck after local durability and + participant turnover; without explicit `--revoke-unprovisioned`, an unqualified + live subject refuses with no cutoff write; reverting a SCIM-on pending + value to SCIM-off current remains supported; a process configured with another + cell's issuer answers 503 to every bearer request, through the router or directly; - during a `loom cell update`, processes on the old and the new configuration both serve, `--finish` releases a narrowed issuer only after no process accepts it, an update that dies after its directory write keeps the widened issuer reserved, and @@ -1991,7 +2043,8 @@ An integration test with two cells and a router in CI's Postgres job: refused, and a cell that starts accepting another cell's issuer takes no placements; - a process matching neither value of its row answers 503 to sign-in, refresh, the OAuth token and device endpoints, the sign-in callback and signup, and its probe route still - echoes prepare's nonce; `--revert` waits for processes matching only `next`; + echoes prepare's nonce; `--revert` requires permanent retirement of every captured + participant, including ones already on current; `update --config` refuses while a different `next` is pending, and an update that died before its `loom.cell_self` write resumes forward; prepare refuses while a pre-N process is live; a configuration change while `attaching` goes through diff --git a/docs/operations/cell-update.md b/docs/operations/cell-update.md new file mode 100644 index 0000000000..6cbfc09f8c --- /dev/null +++ b/docs/operations/cell-update.md @@ -0,0 +1,161 @@ +# Update a cell's configuration or addresses + +`loom cell update` reserves a configuration before the cell accepts it. Run it with +that cell's stores, `LOOM_CELL`, and the router operator credential configured. It +holds the cell's ATTACH lock on one direct Postgres session. Prepare, attach, +restore, another update, and the documented state cleanup cannot run concurrently. +An interrupted command exits unsuccessfully; rerun the same action after correcting +the reported cause. + +## Reserve a configuration + +Write a nonsecret settings file: + +```yaml +version: 1 +settings: + tracker_org: operations +``` + +```bash +loom cell update --cell first --config target.yaml +``` + +The file overlays the CLI's effective settings. Every supplied setting is validated, +including values outside the fingerprint. Secret fields are refused. The command +neither installs the file nor changes the running processes' configuration. It +prints fingerprint digests and changed field names. + +The deployment id, registered public and OAuth host, and bucket and prefix cannot +change. The root-secret fingerprint must match the directory's locked value. + +A SCIM-on target requires this cell's valid SCIM token, OIDC login and required +authentication. The CLI freshly checks active accounts, live refresh sessions and +personal service-account creators against the target provisioning policy before +claiming the directory generation, after durable local acceptance, and after +participant turnover before releasing the reservation. It uses the same bounded +[provisioning census as attach](cell-attach.md); a cached admission decision does +not establish this precondition. Run the CLI with the target's SCIM credentials +and authentication settings configured. + +Provision every required identity before updating. Alternatively, add +`--revoke-unprovisioned` to `--config`, `--finish` or `--revert` to end tokens +issued before the update for subjects that cannot meet its SCIM policy. Accepted +SCIM cutoffs are reported immediately and remain if a later check fails; the +command does not roll them back. Correct the reported cause and rerun the same +action. This flag does not apply to address refresh. Reverting to SCIM-off current +does not require provisioning. + +After attach has begun, the directory reserves both current and next values and +places newly configured orgs before the cell's Postgres accepts next. Overlap checks +include both values of every other attached or attaching cell. A different pending +value refuses. Repeating the identical candidate repairs an interrupted local +write without placing an org twice. + +New configured placements obey the cell's closed flag and quota. Existing winners +in that cell remain valid while closed or full. An org held elsewhere refuses the +whole reservation. A later cell still attaching refuses configured-org changes; +a first cell attaching after begin accepts them through the normal placement gate. + +Before attach/begin has recorded the directory fingerprint, update claims the +registration's generation and changes only the local pair. It places no org and +reserves no issuer or pattern. Attach checks and reserves both local values later. + +## Finish or revert + +Install and roll out the reserved configuration, then run: + +```bash +loom cell update --cell first --finish --wait 600 +``` + +To keep current instead: + +```bash +loom cell update --cell first --revert --wait 600 +``` + +These forms first repair local tenancy for actual same-cell configured placements +from both directory values. Their operator read compares the exact generation, +registration, pair, and root-secret fingerprint, writes an audit and WAL fence, +and waits for synchronous commit. Public lookup's first-cell fallback is not proof +that an immutable placement exists. A missing placement outside the attaching +first cell requires directory metadata recovery. + +Finalization durably narrows local acceptance first. The directory keeps both +reservations until every participant captured by a complete post-cutoff census +has permanently retired or its exact process is proved dead with no outstanding +ClickHouse query. Captured target processes also turn over. A target-serving +replacement must join a live instance, a marked cell heartbeat, and a guarded +participant outside the captured set. A workers-only process cannot satisfy this. + +Every configured ClickHouse replica must answer. Replicated participant tables +synchronize after the local cutoff; the census unions their newest rows without +age filters, sampling, caches, or partial-result limits. Heartbeat expiry, +`drained`, a missing row, an empty query list alone, a request deadline, or deletion +of a pod object does not prove permanent retirement. An uninspectable process or +unreadable topology refuses finalization. + +Online finalization requires the [participant lifecycle baseline](../architecture/cells.md#participant-lifecycle-baseline) +and [controller fencing](../architecture/cells.md#control-mutation-fencing). Positively +terminate or fence every legacy server, worker, one-off command and control writer, +including paused boots, before using this protocol. Prevent old binaries from +returning. Inventory actual process/container termination or independent node +fencing; an expired heartbeat or missing API object is insufficient. The CLI +cannot establish this external inventory from its census. Unknown legacy identities +remain blockers. No command flag asserts or bypasses this prerequisite. + +`--wait` is finite and between 0 and 86400 seconds. Zero still makes a bounded +initial attempt. Local transactions and complete census reads have their own +bounded attempts. If turnover times out, local acceptance stays single and the +directory stays dual. Keep the target replacement serving, resolve the named +participants and rerun the same action. Each rerun takes a fresh census. Retrying +`--config` can widen local acceptance again and requires another cutoff and census +before finalization. All directory placements and local tenancy rows remain after +revert, including orgs introduced by the pending value. + +| Interruption | Recorded state | Recovery | +| -------------------------------------------------------------- | ----------------------------------------- | ---------------------------------------------------- | +| Directory reservation commits before local write | Directory dual; local pair may be old | Repeat config, finish, or revert explicitly | +| Local cutoff commits before turnover or final directory commit | Directory dual; local single | Repeat the selected finish or revert | +| Generation or ATTACH backend changes | An in-flight directory reply may be stale | Stop; a fresh invocation claims again and recomputes | +| Final directory singleton already matches local | Both single | The no-op still proves synchronous local durability | + +## Refresh addresses + +For a later cell: + +```bash +loom cell update --cell second --probe-url https://second-probe.example.test +``` + +For the first cell, roll out the intended `LOOM_ROUTER_UPSTREAM_URL` and +`LOOM_ROUTER_UPSTREAM_PROBE_URL` to **every router first**, then run without an +action flag: + +```bash +loom cell update --cell first +``` + +The contacted router supplies its normalized first-cell addresses. The probe must +echo fresh Postgres, ClickHouse, and configured bucket nonces and the registered +deployment id, bucket and prefix. A main listener cannot substitute for the probe. +The first cell's main upstream must independently report that deployment id. The +router rechecks its configured addresses at commit. A later refresh changes only +its probe URL; it cannot move the registered host, stores or upstream. + +A generation and address comparison excludes stale requests; it does not establish +fleet-wide configuration agreement. An old router can read the current expected +row and reverse the addresses if it still runs the old configuration. Keep the +old endpoints reachable until every router demonstrably refreshes its routing +table or restarts **after** the directory change. A configuration rollout alone +does not flush cached routes. During a directory outage a router can keep its last +routing table indefinitely. The command cannot certify all-router agreement. + +Configuration actions, finish, revert and probe refresh are mutually exclusive. +Probe readiness retries use `--wait`; invalid proofs refuse immediately. Nonce +cleanup uses a value-qualified Postgres delete and a Keeper compare-and-delete. +Bucket cleanup reads and compares this run's value before an unconditional delete. +If the ATTACH backend is lost, a successor can replace the object between that read +and delete and lose its nonce. Rerun the affected proof; bucket cleanup does not +guarantee preservation of a successor's nonce. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 5810ff2d99..2bd191143d 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -334,8 +334,10 @@ python scripts/codegen/docs_reference.py | `loom cell limit` | Close, reopen or bound a cell to new orgs. | `--cell`, `--closed`, `--open`, `--placements-max`, `--router` | | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | +| `loom cell place` | Place an org in this cell and record its tenancy. | `--cell`, `--org`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | | `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--revoke-unprovisioned`, `--router` | +| `loom cell update` | Reserve configuration, finish or revert its rollout, or refresh verified addresses. | `--cell`, `--config`, `--finish`, `--revert`, `--probe-url`, `--revoke-unprovisioned`, `--router`, `--wait` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/llms.txt b/llms.txt index 4f75fd568c..bd67612467 100644 --- a/llms.txt +++ b/llms.txt @@ -108,6 +108,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. - [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [Cell control upgrades and recovery](docs/operations/cell-control.md): Cell control mutations require generation-aware clients and router handlers, and direct access to the cell's Postgres. +- [Update a cell's configuration or addresses](docs/operations/cell-update.md): `loom cell update` reserves a configuration before the cell accepts it. - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 016489264c..bd18707617 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -427,3 +427,30 @@ async def _go() -> T: return result return asyncio.run(_go()) + + +def run_control_connected[T](run: Callable[[CHClient], Awaitable[T]], *, org_free: OrgFree) -> T: + """Connect an operator-only coordinator without admitting model or org work. + + Such a coordinator cannot join the participant set whose retirement it waits + for. It still verifies paired stores and the explicit org-free write gate. + """ + from server.core.ch import get_client + + _declared(None, org_free) + + async def go() -> T: + ch = get_client() + await connect_verified(ch) + try: + await check_acceptance(org_free=org_free) + result = await run(ch) + except BaseException: # Keep the coordinator failure while closing its clients. + with suppress(BaseException): + await disconnect(ch) + raise + else: + await disconnect(ch) + return result + + return asyncio.run(go()) diff --git a/server/cli/cell.py b/server/cli/cell.py index a7b275efcc..b586e12cad 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -476,7 +476,11 @@ def _add_cell_parser(sub: Any) -> None: remove.set_defaults(func=_cmd_remove) from server.cli.cell_attach import add_attach_parser + from server.cli.cell_place import add_place_parser from server.cli.cell_prepare import add_prepare_parser + from server.cli.cell_update import add_update_parser + add_place_parser(csub) add_prepare_parser(csub) add_attach_parser(csub) + add_update_parser(csub) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index efb01f532a..0fbaade144 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -143,12 +143,22 @@ async def admitted() -> T | _Fatal: task.add_done_callback(pg._retrieve_task) return task + def _bind(self, conn: Any) -> bool: + self.conn = conn + return not self.closed + + async def connect(self, *, statement_timeout_ms: int) -> None: + if self.conn is not None or self.closed: + raise _refused("a control attempt cannot open another session") + try: + await pg._connect_dedicated(statement_timeout_ms=statement_timeout_ms, bind=self._bind) + except BaseException: + with contextlib.suppress(BaseException): + await self.close() + raise + async def _open(self, what: str) -> None: - self.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) - if self.closed: - # A connector returning after cancellation cannot publish a usable session. - await self.close() - raise _refused("connection arrived after the control attempt ended") + await self.connect(statement_timeout_ms=60_000) refused = await apg.direct_refusals(self.conn, what) refused += await apg.encoding_refusals(self.conn) if refused: diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py new file mode 100644 index 0000000000..b564ee0cff --- /dev/null +++ b/server/cli/cell_place.py @@ -0,0 +1,184 @@ +"""Directory-first operator placement, followed by a durable local tenancy row.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import sys +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_pg as apg +from server.cli._ch import run_connected, writes +from server.cli.cell_ch import Refused +from server.cli.cell_control import Owner +from server.core import cell, pg +from server.router.directory import PLACE_MAX_S +from server.settings import settings + +# The directory statement bound plus connection and client cleanup. +REQUEST_S = PLACE_MAX_S + 10.0 # not a setting: derived + + +class PlacementRefused(Exception): + """A placement cannot complete with this configuration or directory winner.""" + + +def _bound_s() -> float: + return settings.pg_statement_timeout_ms / 1000 + 1 + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return "--cell names 1 to 32 of a-z, 0-9, -" + if args.cell != settings.cell: + return f"place runs with the cell's own configuration: set LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return "place needs LOOM_STORAGE_AUTHORITY=postgres and the cell's LOOM_PG_DSN" + if cell.named_org(args.org) is None: + return "--org names a nonempty Postgres org" + return cell.org_name_refusal(args.org) + + +async def _identity(conn: Any, name: str) -> tuple[str, dict[str, Any]]: + """Read the full local cell state and this CLI's accepted configuration.""" + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + state, first = await cell.read_state_if_present(conn) + if state != "enforced" or first != name: + raise PlacementRefused( + f"local cell state is {state!r}, naming {first!r}; place needs enforced {name!r}" + ) + deployment = await conn.fetchval("SELECT deployment_id FROM loom.authority") + if not isinstance(deployment, str) or not deployment: + raise PlacementRefused("the local Postgres records no deployment id") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + mine = cell.fingerprint(deployment) + if own is None or not any( + cell.matches(mine, cell.json_object(own[k])) for k in ("current_fp", "next_fp") + ): + raise PlacementRefused("the CLI's configuration matches no local cell fingerprint") + return deployment, cell.as_value(mine) + + +async def _directory( + router: httpx.URL, name: str, org: str, deployment: str, fingerprint: dict[str, Any] +) -> None: + url = cell_cli.at(router, f"/cells/v1/operator/cells/{name}/place") + manager = httpx.AsyncClient(trust_env=router.scheme == "https") + client = await manager.__aenter__() + try: + answer = await client.post( + url, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, + timeout=httpx.Timeout(REQUEST_S, connect=5.0), + ) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) + if posted is None: + raise PlacementRefused(cell_cli.not_the_router(answer)) + if answer.status_code != 200: + raise PlacementRefused(f"the router refused ({answer.status_code}): {posted['error']}") + if posted.get("cell") != name: + raise PlacementRefused( + f"org {org!r} belongs to cell {posted.get('cell')!r}; no local tenancy row written" + ) + if posted.get("deployment_id") != deployment: + raise PlacementRefused("the router answered another deployment id") + + +async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: + """Hold the operator lock through directory placement and an acknowledged local commit. + Local phases and the complete directory exchange have separate deadlines.""" + owner = Owner(exit_s=_bound_s()) + + async def opening() -> None: + await owner.connect(statement_timeout_ms=settings.pg_statement_timeout_ms) + + async def initial() -> tuple[str, dict[str, Any]]: + if refusals := await apg.direct_refusals(owner.conn, "place"): + raise Refused(refusals) + await apg.take_lock(owner.conn, "place") + return await _identity(owner.conn, args.cell) + + async def work() -> dict[str, str]: + deployment, fingerprint = await owner.call(initial, total_s=_bound_s()) + await owner.call( + lambda: _directory(router, args.cell, args.org, deployment, fingerprint), + total_s=REQUEST_S, + ) + + async def local() -> None: + conn = owner.conn + tx = conn.transaction() + await tx.start() + if not await apg.lock_held(conn): + raise PlacementRefused("the operator lock was lost") + if await _identity(conn, args.cell) != (deployment, fingerprint): + raise PlacementRefused( + "the local identity or configuration changed during placement" + ) + await conn.execute( + "INSERT INTO loom.tenancy (org) VALUES ($1) ON CONFLICT DO NOTHING", args.org + ) + # A no-op insert must also flush a primary-only predecessor to the standby. + await conn.execute(pg._DURABLE_MESSAGE) + await tx.commit() + + await owner.call(local, total_s=_bound_s()) + return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + return await owner.run(opening, work, opening_s=_bound_s()) + + +def cmd_place(args: argparse.Namespace) -> int: + if reason := _args_problem(args): + print(f"loom cell place: {reason}", file=sys.stderr) + return 2 + router = cell_cli.operator_router(args) + if isinstance(router, str): + print(f"loom cell place: {router}", file=sys.stderr) + return 2 + + async def run(ch: Any) -> int: + try: + result = await place(args, router) + except asyncio.CancelledError: + print( + "loom cell place: stopped (CancelledError); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + raise + except Exception as e: # noqa: BLE001 — an unknown outcome must name its idempotent retry + print( + f"loom cell place: stopped ({type(e).__name__}: {e}); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + return run_connected( + run, org_free=writes("places an org in the directory and records its local tenancy") + ) + + +def add_place_parser(csub: Any) -> None: + p = csub.add_parser("place", help="Place an org in this cell and record its tenancy.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--org", required=True, help="The org to place") + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_place) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 517a461840..eac6638fd1 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -296,11 +296,14 @@ async def _register( router: httpx.URL, body: dict[str, Any], transport: httpx.AsyncBaseTransport | None, + *, + path: str = "/register", + command: str = "prepare", ) -> dict[str, Any]: """The router's 200 answer, retrying a busy or unreachable probe listener within `--wait`; Refused otherwise.""" token = settings.router_operator_token - url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") + url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}{path}") deadline = time.monotonic() + args.wait manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport @@ -313,12 +316,18 @@ async def _register( except httpx.HTTPError as e: # A rerun answers the row a lost answer may have written. raise Refused( - f"no answer from the router ({type(e).__name__}): run prepare again" + f"no answer from the router ({type(e).__name__}): run {command} again" ) from None answer = cell_cli.router_answer(r, {"cell": args.cell}, cell_cli.OPERATOR_REFUSALS) + if ( + command == "update" + and r.status_code == 404 + and (answer is None or answer.get("code") != "NO_SUCH_CELL") + ): + raise Refused("a router replica is not upgraded for cell update") if answer is None: raise Refused( - f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" + f"{cell_cli.not_the_router(r)}; run {command} again once the router answers" ) if r.status_code == 200: break @@ -326,7 +335,7 @@ async def _register( if code == "BACKEND_UNAVAILABLE": raise Refused( f"the directory did not answer ({answer['error']}); the registration may " - "have landed: run prepare again" + f"have landed: run {command} again" ) if code not in ("PROBE_BUSY", "PROBE_UNREACHABLE"): raise Refused(f"the router refused ({r.status_code}): {answer['error']}") @@ -340,7 +349,7 @@ async def _register( raise Refused( f"the router could not probe the cell within --wait: {answer['error']}" ) - print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) + print(f"loom cell {command}: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) except BaseException: # Client exit preserves the first request or refusal. diff --git a/server/cli/cell_scim.py b/server/cli/cell_scim.py index 762efb20fc..aa0573c347 100644 --- a/server/cli/cell_scim.py +++ b/server/cli/cell_scim.py @@ -262,16 +262,21 @@ async def check( seconds: float, revoke: bool = False, dry_run: bool = False, + target: dict[str, Any] | None = None, report: dict[str, Any] | None = None, log: Callable[[str], None] = lambda _message: None, ) -> None: """Read the intended policy freshly; contract only under the native attach owner.""" - if not required(*own): + if not (required(target) if target is not None else required(*own)): return if not math.isfinite(seconds) or seconds <= 0: raise _refuse("the provisioning census needs a finite positive scan budget") if client is not ch.get_client(): raise _refuse("the census client differs from this process's configured store") + if target is not None and target.get("fields", {}).get("deployment_id") != row.get( + "deployment_id" + ): + raise _refuse("the intended configuration names another deployment") status = report if report is not None else {} status.update(checked=0, revoked=0, unprovisioned=0, complete=False) deadline = time.monotonic() + seconds diff --git a/server/cli/cell_update.py b/server/cli/cell_update.py new file mode 100644 index 0000000000..e60926e33f --- /dev/null +++ b/server/cli/cell_update.py @@ -0,0 +1,129 @@ +"""Operator configuration updates and verified address refresh for a prepared cell.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import sys +from pathlib import Path +from typing import Any + +from server.cli import cell as cell_cli +from server.cli._ch import run_control_connected, writes +from server.cli.cell_prepare import _WAIT_MAX_S +from server.core import cell +from server.settings import settings + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return "--cell names 1 to 32 of a-z, 0-9, -" + if settings.cell != args.cell: + return f"update runs with this cell's configuration: set LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return "update needs LOOM_STORAGE_AUTHORITY=postgres and this cell's LOOM_PG_DSN" + if not math.isfinite(args.wait) or not 0 <= args.wait <= _WAIT_MAX_S: + return f"--wait is finite, nonnegative and at most {_WAIT_MAX_S} seconds" + if sum(bool(value) for value in (args.config, args.finish, args.revert, args.probe_url)) > 1: + return "choose one of --config, --finish, --revert or --probe-url" + if getattr(args, "revoke_unprovisioned", False) and not ( + args.config or args.finish or args.revert + ): + return "--revoke-unprovisioned needs --config, --finish or --revert" + return "" + + +def cmd_update(args: argparse.Namespace) -> int: + if reason := _args_problem(args): + print(f"loom cell update: {reason}", file=sys.stderr) + return 2 + router = cell_cli.operator_router(args) + if isinstance(router, str): + print(f"loom cell update: {router}", file=sys.stderr) + return 2 + + async def run(ch: Any) -> int: + from server.cli.cell_update_run import update + + try: + answer = await update(ch, args, router) + except asyncio.CancelledError: + print( + "loom cell update: interrupted; outcome may be unknown; rerun the same action", + file=sys.stderr, + ) + raise + except Exception as error: # noqa: BLE001 — failures must identify the bounded recovery action. + print( + f"loom cell update: stopped ({type(error).__name__}: {error}); " + "rerun the same action", + file=sys.stderr, + ) + return 1 + print(json.dumps(answer, indent=2, sort_keys=True)) + return 0 + + return run_control_connected( + run, org_free=writes("updates this cell's reserved configuration and verified addresses") + ) + + +def add_update_parser(csub: Any) -> None: + parser = csub.add_parser( + "update", + help="Reserve configuration, finish or revert its rollout, or refresh verified addresses.", + ) + parser.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + action = parser.add_mutually_exclusive_group() + action.add_argument( + "--config", + type=Path, + help=( + "Nonsecret version: 1 settings YAML overlaid on this CLI's effective settings; " + "does not install the file" + ), + ) + action.add_argument( + "--finish", + action="store_true", + help="Promote next after durable local narrowing and permanent participant turnover", + ) + action.add_argument( + "--revert", + action="store_true", + help=( + "Retain current after durable local narrowing and permanent participant " + "turnover; keep placements" + ), + ) + action.add_argument( + "--probe-url", + help=( + "Later cell's verified probe listener URL; no flag refreshes the first cell's " + "router-configured addresses" + ), + ) + parser.add_argument( + "--revoke-unprovisioned", + action="store_true", + help="Cut off pre-update tokens of subjects that cannot satisfy the target SCIM policy", + ) + parser.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + parser.add_argument( + "--wait", + type=float, + default=600, + help=( + "Seconds to wait for probe readiness or participant turnover (0 to 86400, default: 600)" + ), + ) + parser.epilog = ( + "Finish/revert require foundation-capable runtimes and completed " + "positive legacy-process turnover. A census cannot verify that external " + "inventory. Address refresh requires every router updated first; retain " + "old endpoints until every router refreshes or restarts after the " + "directory change." + ) + parser.set_defaults(func=cmd_update) diff --git a/server/cli/cell_update_addresses.py b/server/cli/cell_update_addresses.py new file mode 100644 index 0000000000..9d9812ae5e --- /dev/null +++ b/server/cli/cell_update_addresses.py @@ -0,0 +1,64 @@ +"""Refresh proved probe/upstream addresses without claiming fleet-wide cache agreement.""" + +from __future__ import annotations + +import json +from typing import Any + +from server.cli import cell_control, cell_update_probe +from server.router.app import _cell_address +from server.router.directory import CONTROL_FIELDS + + +def normalise(value: str) -> str: + address = _cell_address(value) + if not address: + raise cell_control._refused("probe address is an HTTP(S) host-only URL") + return address + + +async def refresh(run: Any) -> dict[str, Any]: + if run.row["is_first"]: + if run.args.probe_url: + raise cell_control._refused( + "the first cell refreshes router-configured addresses without --probe-url" + ) + proposal = await run.call("/addresses") + if json.dumps(cell_control.preconditions(proposal), sort_keys=True) != json.dumps( + cell_control.preconditions(run.row), sort_keys=True + ): + raise cell_control._refused( + "directory ownership changed while reading router addresses" + ) + probe_url = normalise(proposal.get("router_upstream_probe_url") or "") + upstream_url = normalise(proposal.get("router_upstream_url") or "") + else: + if not run.args.probe_url: + raise cell_control._refused("a later cell probe refresh requires --probe-url") + probe_url, upstream_url = normalise(run.args.probe_url), run.row["upstream_url"] + answer = await cell_update_probe.post( + run, "/addresses", {"probe_url": probe_url, "upstream_url": upstream_url} + ) + if ( + answer["mutation_generation"] != run.row["mutation_generation"] + or any( + json.dumps(answer[name], sort_keys=True) != json.dumps(run.row[name], sort_keys=True) + for name in CONTROL_FIELDS + if name not in ("probe_url", "upstream_url") + ) + or (answer["probe_url"], answer["upstream_url"]) != (probe_url, upstream_url) + ): + raise cell_control._refused( + "address acknowledgement differs from the proved addresses; outcome unknown" + ) + return { + "cell": run.args.cell, + "probe_url": probe_url, + "upstream_url": upstream_url, + "router_refresh_required": bool(run.row["is_first"]), + "next_step": ( + "Keep old endpoints reachable until every router refreshes its routing " + "table or restarts after this directory update; this command cannot " + "prove all-router agreement" + ), + } diff --git a/server/cli/cell_update_config.py b/server/cli/cell_update_config.py new file mode 100644 index 0000000000..2e82113b42 --- /dev/null +++ b/server/cli/cell_update_config.py @@ -0,0 +1,82 @@ +"""An explicit nonsecret settings overlay for a cell's target configuration.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pydantic import ValidationError +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource + +from server.code.domain.index import repos +from server.core import cell +from server.router import fingerprints +from server.settings import Settings +from server.settings._file import SettingsFileError, read_settings_file +from server.settings._live import coerce + + +class _Target(Settings): + """Validate the supplied complete model without rereading external sources.""" + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (init_settings,) + + +def read_target(path: Path, current: Settings) -> Settings: + """Overlay FILE on the CLI's effective settings; install neither the file nor values.""" + try: + values = read_settings_file(str(path), set(Settings.model_fields)) + except Exception: # noqa: BLE001 — parser failures must not expose target file values. + # YAML constructors can raise builtin errors that contain their input. + raise SettingsFileError( + f"{path}: invalid target settings file; use version: 1, unique setting names " + "and nonsecret values" + ) from None + checked: dict[str, Any] = {} + for name, value in values.items(): + try: + checked[name] = coerce(Settings, name, value) + except ValidationError: + raise SettingsFileError(f"{path}: {name} has an invalid target value") from None + try: + return _Target(**(current.model_dump() | checked)) + except ValidationError: + # Validators can embed supplied values in their own messages despite hidden inputs. + raise SettingsFileError( + f"{path}: invalid target configuration; check the setting constraints" + ) from None + + +def target_fingerprint( + target: Settings, deployment_id: str, registered: dict[str, Any] +) -> dict[str, Any]: + """Validate the candidate without admitting it or changing process settings.""" + value = cell.as_value( + cell.fingerprint_for(target, deployment_id, repos.operator_orgs_for(target)) + ) + if reason := fingerprints.registration_refusal(value, registered): + raise ValueError(reason) + if value["fields"]["scim"]: + from server.auth.scim_routes import scim_settings_problem + + problem = scim_settings_problem( + target.scim_token, + target.oauth_login_provider, + target.auth_actually_required(), + cell_name=registered["cell"], + first=(registered["is_first"], registered["cell"]), + ) + if problem: + raise ValueError(problem) + if reason := fingerprints.placement_refusal(registered["cell"], "candidate", value): + raise ValueError(reason) + return value diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py new file mode 100644 index 0000000000..26a3fa8c73 --- /dev/null +++ b/server/cli/cell_update_participants.py @@ -0,0 +1,256 @@ +"""A post-cutoff participant census and permanent retirement barrier for update.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import time +from typing import Any, cast +from uuid import UUID + +from server.cli import cell_control, cell_scan, cell_writers +from server.cli.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology +from server.core import cell, ch_names, instances +from server.core.ch import query_tag +from server.core.provider_process import ( + certified, + has_certificate, + participant_tag, + proves_stopped, +) +from server.insights.store import provider_participants as participants + + +class CompleteReads: + """Pass complete uncached read settings to the existing instance reader.""" + + def __init__(self, ch: Any) -> None: + self.ch = ch + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return cast( + list[dict[str, Any]], + await self.ch.query_all(complete(sql, sequential=1), params or {}), + ) + + +def complete(sql: str, *, sequential: int = 0) -> str: + return ( + sql + + " " + + WHOLE_SQL + + ", skip_unavailable_shards=0, prefer_localhost_replica=1, " + + f"select_sequential_consistency={sequential}" + ) + + +def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + # The latest winner cannot hide a conflicting older version from another replica. + seen: dict[tuple[str, int], tuple[str, str]] = {} + for row in rows: + try: + identity = str(UUID(str(row["participant_id"]))) + version = row["version"] + state, proof = row["state"], row["host_proof"] + except (KeyError, TypeError, ValueError): + raise cell_control._refused("participant census contains an unreadable row") from None + if ( + type(version) is not int + or version <= 0 + or state not in ("running", "drained", "retired") + or not isinstance(proof, str) + ): + raise cell_control._refused("participant census contains an unreadable row") + previous = out.get(identity) + observed = seen.get((identity, version)) + if observed is not None and observed != (state, proof): + raise cell_control._refused(f"participant {identity} has conflicting equal versions") + seen[(identity, version)] = (state, proof) + if previous is None or int(previous["version"]) < version: + out[identity] = dict(row) + return out + + +def capture(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return { + identity: row + for identity, row in latest(rows).items() + if row["state"] != "retired" or not has_certificate(row["host_proof"]) + } + + +async def census(ch: Any) -> dict[str, dict[str, Any]]: + """Synchronize after the durable cutoff; every configured replica must answer.""" + topology = await read_topology(ch) + if topology.cluster and not topology.entries: + raise cell_control._refused("participant census cannot resolve the configured cluster") + if problems := topology.refusals(): + raise Refused(problems) + names = await cell_writers.host_names(ch, topology) if topology.multi else {} + if problems := (await cell_writers.own_refusals(ch, topology, names))[0]: + raise Refused(problems) + database = ch_names.database() + tables = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, engine FROM {source:tables} WHERE database={db:String} AND " + "name='_provider_participants'" + ), + {"db": database}, + ) + if not tables or any(len(rows) != 1 for rows in tables.values()): + raise cell_control._refused("participant table is missing from a configured replica") + engines = {str(rows[0]["engine"]) for rows in tables.values()} + if len(engines) != 1 or not next(iter(engines)).endswith("MergeTree"): + raise cell_control._refused( + "participant replicas have unreadable or different table engines" + ) + engine = next(iter(engines)) + if engine.startswith("Replicated"): + rows = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, database, table, zookeeper_path, replica_name, " + "replica_is_active FROM {source:replicas} WHERE database={db:String} AND " + "table='_provider_participants'" + ), + {"db": database}, + ) + if any(len(value) != 1 for value in rows.values()) or cell_writers.extra_replicas( + row for value in rows.values() for row in value + ): + raise cell_control._refused("participant replica set includes an unverified replica") + budget = cell_scan.Budget(cell_control.TRANSACTION_S) + await cell_scan._system( + ch, + f"SYSTEM SYNC REPLICA{cell_scan._on_cluster(topology)} " + f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", + topology, + budget, + "participant census", + ) + source = f"{quote(database)}.`_provider_participants`" + params = {} + if topology.multi: + source = f"clusterAllReplicas({{cluster:String}}, {source})" + params["cluster"] = topology.cluster + sql = f"SELECT participant_id, state, version, host_proof FROM {source} FINAL" + rows = await ch.query_all(complete(sql, sequential=int(engine.startswith("Shared"))), params) + return latest(rows) + + +async def stopped(ch: Any, initial: dict[str, Any], current: dict[str, Any] | None) -> bool: + if current is not None: + try: + same_identity = certified(initial["host_proof"]) == current["host_proof"] + except ValueError: + same_identity = False + if ( + current["state"] == "retired" + and has_certificate(current["host_proof"]) + and same_identity + and int(current["version"]) > int(initial["version"]) + ): + return True + if current is not None: + try: + if certified(initial["host_proof"]) != certified(current["host_proof"]): + return False + except ValueError: + if initial["host_proof"] != current["host_proof"]: + return False + if not proves_stopped(initial["host_proof"]): + return False + # Never use the model protocol's missing-cluster local fallback as death proof. + row = await ch.query_one( + complete(participants.running_sql()), + {"tag": participant_tag(initial["participant_id"])}, + ) + return row is not None and type(row.get("running")) is int and row["running"] == 0 + + +async def replacement( + ch: Any, run: Any, rows: dict[str, dict[str, Any]], captured: set[str], target: dict[str, Any] +) -> bool: + reader = CompleteReads(ch) + live = await instances.read_live(reader, all_deployments=True) + reported = await reader.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": [str(row["instance_id"]) for row in live]}, + ) + by_instance = {str(row["instance_id"]): row for row in reported} + for instance in live: + beat = by_instance.get(str(instance["instance_id"])) + if not beat or not instance["serving"] or beat["cell"] != run.args.cell: + continue + try: + value = json.loads(beat["fingerprint"]) + identity = str(UUID(value["participant_id"])) + except (ValueError, TypeError, KeyError): + continue + participant = rows.get(identity) + if ( + identity in captured + or participant is None + or participant["state"] == "retired" + or not has_certificate(participant["host_proof"]) + ): + continue + if ( + value.get("serves") is not True + or value.get("marked") is not True + or (run.row["is_first"] and value.get("router") is not True) + ): + continue + if beat["tenancy_mode"] not in ("recording", "enforced") or not isinstance( + value.get("fields"), dict + ): + continue + if cell.matches(value["fields"], target): + return True + return False + + +async def settle(run: Any, target: dict[str, Any]) -> None: + """Fix one complete captured set; expiry never removes an obligation.""" + with query_tag("cell.update.participants"): + initial = await run.owner.call(lambda: census(run.ch), total_s=cell_control.TRANSACTION_S) + captured = capture(list(initial.values())) + done: set[str] = set() + deadline = time.monotonic() + run.args.wait + while True: + await cell_control.check_lock(run.owner.conn) + + async def poll() -> bool: + rows = await census(run.ch) + for identity, original in captured.items(): + if identity not in done and await stopped(run.ch, original, rows.get(identity)): + done.add(identity) + return await replacement(run.ch, run, rows, set(captured), target) + + live = await run.owner.call(poll, total_s=cell_control.TRANSACTION_S) + remaining = sorted(set(captured) - done) + if not remaining and live: + return + if time.monotonic() >= deadline: + blocked = ", ".join(remaining) or "no captured participant remains" + raise cell_control._refused( + f"participant turnover incomplete ({blocked}); " + f"serving target replacement={live}. " + "Directory stays dual and local acceptance stays single. Online finalization " + "requires foundation code and positive legacy-process turnover; this census " + "cannot verify that external inventory" + ) + print( + f"loom cell update: waiting for participants {', '.join(remaining) or 'none'}; " + f"serving target replacement={live}", + file=sys.stderr, + ) + await asyncio.sleep(min(1, max(0, deadline - time.monotonic()))) diff --git a/server/cli/cell_update_pg.py b/server/cli/cell_update_pg.py new file mode 100644 index 0000000000..9be29d2571 --- /dev/null +++ b/server/cli/cell_update_pg.py @@ -0,0 +1,126 @@ +"""Update snapshots and acceptance writes on the exact ATTACH-owning backend.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from server.cli import cell_control +from server.core import cell +from server.router import fingerprints + + +@dataclass(frozen=True) +class State: + deployment_id: str + is_first: bool + cell: str + mode: str + current: dict[str, Any] + pending: dict[str, Any] | None + + def identity(self) -> tuple[Any, ...]: + return ( + self.deployment_id, + self.is_first, + self.cell, + self.mode, + json.dumps((self.current, self.pending), sort_keys=True), + ) + + +async def read(conn: Any, name: str) -> State: + row = await conn.fetchrow( + "SELECT a.deployment_id, f.is_first, f.cell, m.mode, s.current_fp, s.next_fp " + "FROM loom.authority a " + "LEFT JOIN loom.cell_first f ON true LEFT JOIN loom.tenancy_mode m ON true " + "LEFT JOIN loom.cell_self s ON true" + ) + if ( + row is None + or not row["deployment_id"] + or type(row["is_first"]) is not bool + or row["cell"] != name + or row["mode"] not in ("recording", "enforced") + ): + raise cell_control._refused( + "update needs complete prepared state naming this cell; prepare or restore it first" + ) + current, pending = cell.json_object(row["current_fp"]), cell.json_object(row["next_fp"]) + if not fingerprints.readable(current) or ( + pending is not None and not fingerprints.readable(pending) + ): + raise cell_control._refused("update needs readable local current/next fingerprints") + assert current is not None + return State( + str(row["deployment_id"]), + row["is_first"], + name, + row["mode"], + current, + pending, + ) + + +def paired(state: State, registered: dict[str, Any]) -> None: + if ( + state.deployment_id != registered["deployment_id"] + or state.is_first != registered["is_first"] + ): + raise cell_control._refused( + "the prepared local identity differs from the directory registration" + ) + if registered["current_fp"] is not None: + accepted = { + json.dumps(value, sort_keys=True) + for value in (registered["current_fp"], registered["next_fp"]) + if value is not None + } + if any( + json.dumps(value, sort_keys=True) not in accepted + for value in (state.current, state.pending) + if value is not None + ): + raise cell_control._refused( + "local fingerprints differ from the directory reservation; restore the " + "recorded pair" + ) + + +async def write( + owner: cell_control.Owner, + expected: State, + registered: dict[str, Any], + current: dict[str, Any], + pending: dict[str, Any] | None, + placed_orgs: list[str], +) -> State: + cell_control.preconditions(registered) + + async def change() -> State: + conn = owner.conn + await cell_control.paired(conn, registered) + actual = await read(conn, expected.cell) + if actual.identity() != expected.identity(): + raise cell_control._refused( + "local acceptance changed; stop and rerun with fresh observations" + ) + if placed_orgs: + await conn.execute( + "INSERT INTO loom.tenancy(org) SELECT org FROM unnest($1::text[]) AS org " + "ON CONFLICT DO NOTHING", + placed_orgs, + ) + result = await conn.execute( + "UPDATE loom.cell_self SET current_fp=$1, next_fp=$2, updated_at=now() WHERE id=1", + current, + pending, + ) + if result != "UPDATE 1": + raise cell_control._refused( + "local acceptance row disappeared; restore it before updating" + ) + return await read(conn, expected.cell) + + return await owner.transaction(change) diff --git a/server/cli/cell_update_probe.py b/server/cli/cell_update_probe.py new file mode 100644 index 0000000000..d102969225 --- /dev/null +++ b/server/cli/cell_update_probe.py @@ -0,0 +1,44 @@ +"""Fresh store proof with bounded retries and prepare's store-specific nonce cleanup.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from server.cli import cell_control, cell_prepare +from server.router.app import REGISTER_ANSWER_MAX_S + +# Read/probe/commit bounds already used by prepare, plus its reply margin. +ANSWER_S = REGISTER_ANSWER_MAX_S + 10.0 + + +async def post(run: Any, path: str, body: dict[str, Any]) -> dict[str, Any]: + await cell_control.check_lock(run.owner.conn) + nonces = cell_prepare._new_nonces() + + async def clean() -> None: + await cell_prepare._delete_nonces(run.ch, nonces) + + try: + await run.owner.call( + lambda: cell_prepare._write_nonces(run.ch, nonces), total_s=cell_control.TRANSACTION_S + ) + async with asyncio.timeout(run.args.wait + ANSWER_S): + answer = await cell_prepare._register( + run.args, + run.router, + {**body, "nonces": nonces, **cell_control.preconditions(run.row)}, + run.transport, + path=path, + command="update", + ) + await cell_control.check_lock(run.owner.conn) + cell_control.preconditions(answer) + except BaseException: + try: + await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + except BaseException: # noqa: BLE001 — preserve the original operation failure. + pass + raise + await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + return answer diff --git a/server/cli/cell_update_run.py b/server/cli/cell_update_run.py new file mode 100644 index 0000000000..c6accbd784 --- /dev/null +++ b/server/cli/cell_update_run.py @@ -0,0 +1,247 @@ +"""Directory-first reservation and local-first narrowing under one retained owner.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import httpx + +from server.cli import cell_control, cell_scim, cell_update_config +from server.cli import cell_update_pg as local +from server.cli import cell_update_probe as probe +from server.core import cell, deployment_config, pg_profile +from server.router import fingerprints +from server.router.directory import CONTROL_FIELDS +from server.settings import settings + + +def same(a: Any, b: Any) -> bool: + return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + + +class Run: + def __init__( + self, ch: Any, args: argparse.Namespace, router: httpx.URL, transport: Any + ) -> None: + self.ch, self.args, self.router, self.transport = ch, args, router, transport + self.owner: cell_control.Owner + self.row: dict[str, Any] = {} + self.state: local.State + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + return await cell_control.request( + self.router, + self.args.cell, + self.transport, + path, + body, + method="GET" if body is None else "POST", + ) + + async def durable(self) -> local.State: + async def read() -> local.State: + if await pg_profile.standby_now() != "synchronous": + raise cell_control._refused("update requires an attached synchronous standby") + found = await local.read(self.owner.conn, self.args.cell) + if await pg_profile.standby_now() != "synchronous": + raise cell_control._refused("the synchronous standby disappeared during update") + return found + + return await self.owner.transaction(read) + + async def start(self, owner: cell_control.Owner) -> dict[str, Any]: + self.owner = owner + self.state = await self.durable() + self.row = await self.call("") + local.paired(self.state, self.row) + self.target: dict[str, Any] | None = None + if self.args.config: + self.target = cell_update_config.target_fingerprint( + cell_update_config.read_target(Path(self.args.config), settings), + self.state.deployment_id, + self.row, + ) + if self.args.config or self.args.finish or self.args.revert: + _action, _recorded, _current, _pending, target, _desired, _next = self.selection() + await self.scim_precondition(target) + elif self.row["is_first"] and self.args.probe_url: + raise cell_control._refused( + "the first cell refreshes router-configured addresses without --probe-url" + ) + elif not self.row["is_first"] and not self.args.probe_url: + raise cell_control._refused("a later cell probe refresh requires --probe-url") + self.row = await cell_control.claim(owner.conn, self.call, self.row) + local.paired(self.state, self.row) + if self.args.config or self.args.finish or self.args.revert: + return await self.configuration() + from server.cli.cell_update_addresses import refresh + + return await refresh(self) + + async def scim_precondition(self, target: dict[str, Any]) -> None: + from server.cli.cell_attach import _SCAN_BUDGET_S + + await cell_scim.check( + self.ch, + self.owner.conn, + self.row, + (self.state.current, self.state.pending), + target=target, + seconds=_SCAN_BUDGET_S, + revoke=getattr(self.args, "revoke_unprovisioned", False), + log=lambda message: print(f"loom cell update: {message}", file=sys.stderr), + ) + + def check_answer( + self, answer: dict[str, Any], current: dict[str, Any], pending: dict[str, Any] | None + ) -> None: + cell_control.preconditions(answer) + if ( + answer["mutation_generation"] != self.row["mutation_generation"] + or any( + not same(answer[name], self.row[name]) + for name in CONTROL_FIELDS + if name not in ("current_fp", "next_fp") + ) + or not same((answer["current_fp"], answer["next_fp"]), (current, pending)) + ): + raise cell_control._refused( + "directory update acknowledgement differs from its intended result; outcome unknown" + ) + + async def held_orgs(self, current: dict[str, Any], pending: dict[str, Any] | None) -> list[str]: + orgs = sorted( + { + org + for value in (current, pending) + if value is not None + for org in value["fields"].get("configured_orgs", []) + } + ) + result = await cell_control.mutate( + self.owner.conn, + self.call, + self.row, + "/configuration/placements", + {"orgs": orgs, "keys_fp": deployment_config.fingerprints(settings)["keys_fp"]}, + ) + held = result.get("orgs") + if ( + not isinstance(held, list) + or any(type(org) is not str for org in held) + or len(set(held)) != len(held) + or not set(held) <= set(orgs) + ): + raise cell_control._refused( + "directory returned an unreadable configured-placement proof" + ) + if set(held) != set(orgs) and not ( + self.row["is_first"] and self.row["state"] == "attaching" + ): + raise cell_control._refused( + "configured placement is missing; restore directory metadata before updating" + ) + return held + + async def publish(self, target: dict[str, Any], action: str) -> dict[str, Any]: + keys = deployment_config.fingerprints(settings)["keys_fp"] + return await probe.post( + self, "/configuration", {"action": action, "target_fp": target, "keys_fp": keys} + ) + + def selection( + self, + ) -> tuple[ + str, + bool, + dict[str, Any], + dict[str, Any] | None, + dict[str, Any], + dict[str, Any], + dict[str, Any] | None, + ]: + action = "config" if self.args.config else "finish" if self.args.finish else "revert" + recorded = self.row["current_fp"] is not None + current = self.row["current_fp"] if recorded else self.state.current + pending = self.row["next_fp"] if recorded else self.state.pending + target = ( + self.target + if action == "config" + else pending + if action == "finish" and pending is not None + else current + ) + assert target is not None + desired, nxt = fingerprints.configuration_pair(current, pending, target, action) + if reason := fingerprints.registration_pair_refusal(self.row, desired, nxt): + raise cell_control._refused(reason) + return action, recorded, current, pending, target, desired, nxt + + async def configuration(self) -> dict[str, Any]: + action, recorded, current, pending, target, desired, nxt = self.selection() + old_pending = pending is not None + if action == "config" and recorded: + answer = await self.publish(target, action) + self.check_answer(answer, desired, nxt) + self.row = answer + held = await self.held_orgs(self.row["current_fp"], self.row["next_fp"]) if recorded else [] + self.state = await local.write(self.owner, self.state, self.row, desired, nxt, held) + self.state = await self.durable() + if not same((self.state.current, self.state.pending), (desired, nxt)): + raise cell_control._refused("local acceptance differs after its durable update") + await self.scim_precondition(target) + if action != "config" and old_pending: + from server.cli.cell_update_participants import settle + + await settle(self, target) + self.state = await self.durable() + if not same((self.state.current, self.state.pending), (target, None)): + raise cell_control._refused("local cutoff changed while participants turned over") + await self.scim_precondition(target) + # A fresh full-row read refuses a delayed claim before final reservation release. + latest = await self.call("") + if not same(cell_control.preconditions(latest), cell_control.preconditions(self.row)): + raise cell_control._refused( + "directory ownership or reservation changed; rerun with a fresh census" + ) + if recorded: + answer = await self.publish(target, action) + self.check_answer(answer, desired, None) + self.row = answer + elif action != "config" and recorded: + answer = await self.publish(target, action) + self.check_answer(answer, desired, None) + self.row = answer + return { + "cell": self.args.cell, + "action": action, + "directory_reserved": recorded, + "current": cell.digest(desired["fields"]), + "next": None if nxt is None else cell.digest(nxt["fields"]), + "changed_fields": sorted( + name + for name in set(current["fields"]) | set(target["fields"]) + if not same( + current["fields"].get(name, cell.DEFAULTS.get(name)), + target["fields"].get(name, cell.DEFAULTS.get(name)), + ) + ), + "rollout": ( + "reserved; roll out the target configuration before finish" + if recorded + else "local pair prepared; attach must reserve both configurations" + ) + if action == "config" and nxt is not None + else "complete", + } + + +async def update( + ch: Any, args: argparse.Namespace, router: httpx.URL, transport: Any = None +) -> dict[str, Any]: + run = Run(ch, args, router, transport) + return await cell_control.run("update", run.start) diff --git a/server/code/domain/index/repos.py b/server/code/domain/index/repos.py index 3c6bf18e54..1176a18e60 100644 --- a/server/code/domain/index/repos.py +++ b/server/code/domain/index/repos.py @@ -42,7 +42,7 @@ from typing import Any from server.core.errors import BackendUnavailableError, ValidationError -from server.settings import settings +from server.settings import Settings, settings from server.settings.catalog import GIT_PROXY_IDENTITY logger = logging.getLogger("loom.code_index.repos") @@ -186,27 +186,27 @@ def derived_git_dir(repo: str, root: str = CLONE_ROOT, *, fallback: str = "") -> return os.path.join(root, f"{name}.git") if name else "" -def _scalar_repo() -> RepoConfig: +def _scalar_repo(config: Settings) -> RepoConfig: """The scalar settings as one entry. The git dir and remote, when unset, are derived from `code_repo`; with no repository they stay empty (`configured_repos()` then lists no scalar entry).""" - repo = str(settings.code_repo).strip() + repo = str(config.code_repo).strip() return RepoConfig( - org=settings.code_org, - namespace=settings.code_namespace, + org=config.code_org, + namespace=config.code_namespace, repo=repo, - git_dir=settings.code_git_dir or derived_git_dir(repo), - git_remote=settings.code_git_remote or derived_git_remote(repo), - git_branch=settings.code_git_branch, - path_prefixes=_prefixes(settings.code_path_prefixes) or WHOLE_TREE, - github_token=settings.code_github_token, - org_readable=bool(settings.code_org_readable), - public=bool(settings.code_repo_public), - scip_checkout_dir=settings.code_scip_checkout_dir, - git_extra_branches=str(settings.code_git_extra_branches), - git_fetch_pr_heads=bool(settings.code_git_fetch_pr_heads), - exclude=_prefixes(settings.code_ingest_exclude), - ignore_file=str(settings.code_ingest_ignore_file), + git_dir=config.code_git_dir or derived_git_dir(repo), + git_remote=config.code_git_remote or derived_git_remote(repo), + git_branch=config.code_git_branch, + path_prefixes=_prefixes(config.code_path_prefixes) or WHOLE_TREE, + github_token=config.code_github_token, + org_readable=bool(config.code_org_readable), + public=bool(config.code_repo_public), + scip_checkout_dir=config.code_scip_checkout_dir, + git_extra_branches=str(config.code_git_extra_branches), + git_fetch_pr_heads=bool(config.code_git_fetch_pr_heads), + exclude=_prefixes(config.code_ingest_exclude), + ignore_file=str(config.code_ingest_ignore_file), ) @@ -215,7 +215,7 @@ def default_git_dir() -> str: no scalar repository is configured or `configured_repos()` does not list it (a namespace added from the page skips it). The fallback of a read whose namespace has no entry.""" - base = _scalar_repo() + base = _scalar_repo(settings) listed = ( c for c in configured_repos() @@ -367,7 +367,7 @@ def configured_repos() -> list[RepoConfig]: degrades to the scalar single-repo config with a loud log (a typo must not silently stop all indexing) — loud once, then a gauge.""" require_registry_read() - return _with_registry(_configured_from_env()) + return _with_registry(_operator_entries()) def require_registry_read() -> None: @@ -399,7 +399,12 @@ def operator_orgs() -> set[str]: """The orgs the operator's entries name (`LOOM_CODE_REPOS_JSON`, else the scalar repository), without the self-service registry's: a cell's configured orgs (`server.core.cell`).""" - return {c.org for c in _configured_from_env() if c.org} + return {c.org for c in _operator_entries() if c.org} + + +def operator_orgs_for(config: Settings) -> set[str]: + """Configured orgs without changing registry state or the running diagnostics.""" + return {c.org for c in _configured_from_env(config, {}) if c.org} def config_entries() -> list[RepoConfig]: @@ -410,7 +415,7 @@ def config_entries() -> list[RepoConfig]: restricted through an access row reads as its configuration says until then. Never for reading, fetching or indexing a repository: `tests/unit/test_repo_snapshot_contract.py` pins its callers.""" - return _with_registry(_configured_from_env()) if registry_unread() else configured_repos() + return _with_registry(_operator_entries()) if registry_unread() else configured_repos() def config_entry(org: str, namespace: str) -> RepoConfig | None: @@ -468,10 +473,10 @@ def _with_registry(env_repos: list[RepoConfig]) -> list[RepoConfig]: return out + extra -def _scalar_entries(errors: dict[str, str]) -> list[RepoConfig]: +def _scalar_entries(config: Settings, errors: dict[str, str]) -> list[RepoConfig]: """The scalar entry, or none: an empty `code_repo` configures no repository, and a repository without its org or namespace is a configuration error, not an entry.""" - base = _scalar_repo() + base = _scalar_repo(config) if not base.repo: return [] missing = [ @@ -488,12 +493,18 @@ def _scalar_entries(errors: dict[str, str]) -> list[RepoConfig]: return [base] -def _configured_from_env() -> list[RepoConfig]: - raw = settings.code_repos_json.strip() +def _operator_entries() -> list[RepoConfig]: errors: dict[str, str] = {} + result = _configured_from_env(settings, errors) + _publish_config_errors(errors) + return result + + +def _configured_from_env(config: Settings, errors: dict[str, str]) -> list[RepoConfig]: + """Parse explicit operator settings, collecting diagnostics without publishing them.""" + raw = config.code_repos_json.strip() if not raw: - scalar = _scalar_entries(errors) - _publish_config_errors(errors) + scalar = _scalar_entries(config, errors) return scalar try: entries = json.loads(raw) @@ -504,10 +515,9 @@ def _configured_from_env() -> list[RepoConfig]: "LOOM_CODE_REPOS_JSON is not a JSON array — falling back to the " "scalar single-repo config" ) - scalar = _scalar_entries(errors) - _publish_config_errors(errors) + scalar = _scalar_entries(config, errors) return scalar - base = _scalar_repo() + base = _scalar_repo(config) repos: list[RepoConfig] = [] seen: set[str] = set() for i, e in enumerate(entries): @@ -598,8 +608,7 @@ def _configured_from_env() -> list[RepoConfig]: ) ) if not repos: - repos = _scalar_entries(errors) - _publish_config_errors(errors) + repos = _scalar_entries(config, errors) return repos diff --git a/server/core/cell.py b/server/core/cell.py index 87fb824483..eb996a5fcc 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -46,7 +46,7 @@ from server.core import pg, readiness, scim_tokens from server.core.errors import caller_text -from server.settings import settings +from server.settings import Settings, settings from server.settings.catalog import is_http_url log = logging.getLogger(__name__) @@ -180,33 +180,38 @@ def _code_repo_orgs() -> set[str]: def fingerprint(deployment_id: str) -> dict[str, Any]: """This process's fingerprint: what the cell accepts, which the directory checks against every other cell, and the stores it serves from.""" - public = settings.public_base_url - end_users = settings.end_user_issuers() - bucket = settings.document_s3_bucket + return fingerprint_for(settings, deployment_id, _code_repo_orgs()) + + +def fingerprint_for(config: Settings, deployment_id: str, code_orgs: set[str]) -> dict[str, Any]: + """The same fingerprint for an explicit, validated configuration.""" + public = config.public_base_url + end_users = config.end_user_issuers() + bucket = config.document_s3_bucket configured = { o for o in ( - settings.oidc_jit_org, - settings.self_signup_org if settings.self_signup == "open" else "", - settings.code_org, - settings.tracker_org, - *_code_repo_orgs(), + config.oidc_jit_org, + config.self_signup_org if config.self_signup == "open" else "", + config.code_org, + config.tracker_org, + *code_orgs, ) if o } return { "public_host": normalise_host(public), - "oauth_issuer_host": normalise_host(settings.oauth_issuer or public), - "oidc_issuer": settings.oidc_issuer.rstrip("/"), - "oidc_client_id": settings.oidc_client_id, - "scim": bool(settings.scim_token.strip()), - "bearer_issuers": sorted(settings.trusted_issuers() | set(end_users)), + "oauth_issuer_host": normalise_host(config.oauth_issuer or public), + "oidc_issuer": config.oidc_issuer.rstrip("/"), + "oidc_client_id": config.oidc_client_id, + "scim": bool(config.scim_token.strip()), + "bearer_issuers": sorted(config.trusted_issuers() | set(end_users)), "configured_orgs": sorted(configured), "end_user_orgs": sorted({o for spec in end_users.values() for o in spec.orgs}), "deployment_id": deployment_id, "bucket": bucket, # Without a bucket the prefix stores nothing and takes part in no check. - "bucket_prefix": settings.document_s3_prefix if bucket else "", + "bucket_prefix": config.document_s3_prefix if bucket else "", } @@ -428,10 +433,9 @@ def _unset_reason(view: _View) -> str: ) -# The fingerprint fields a CLI's gate judges: the stores it would write. A CLI verifies -# no token and its fingerprint never reaches the router, so the rest of the servers' -# fingerprint is not its to match (docs/architecture/cells.md, "The CLI declares its -# orgs"). +# An org-declaring CLI's gate checks the stores it would write: it verifies no token +# and sends no fingerprint to the router. Operator placement checks the full fingerprint +# separately (docs/architecture/cells.md, "The CLI declares its orgs"). _STORE_FIELDS = ("deployment_id", "bucket", "bucket_prefix") diff --git a/server/core/org_patterns.py b/server/core/org_patterns.py new file mode 100644 index 0000000000..0b7d89551d --- /dev/null +++ b/server/core/org_patterns.py @@ -0,0 +1,19 @@ +"""The exact-name and trailing-star org patterns used by end-user issuers.""" + +from __future__ import annotations + + +def matches(pattern: str, org: str) -> bool: + """Match one literal org; only the pattern's trailing star has special meaning.""" + return org.startswith(pattern[:-1]) if pattern.endswith("*") else org == pattern + + +def overlap(a: str, b: str) -> bool: + """Whether two exact names or prefix patterns can match the same org.""" + if a.endswith("*") and b.endswith("*"): + return a[:-1].startswith(b[:-1]) or b[:-1].startswith(a[:-1]) + if a.endswith("*"): + return matches(a, b) + if b.endswith("*"): + return matches(b, a) + return a == b diff --git a/server/core/pg.py b/server/core/pg.py index ab42f11693..725200e6f4 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -632,13 +632,23 @@ def may_have_committed(e: BaseException) -> bool: return isinstance(e, PostgresUnavailableError) and bool(e.extra.get("outcome_unknown")) -async def _connect_dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> Any: - """Construct, initialize and hand off one process-attributed application session.""" +async def _connect_dedicated( + *, + statement_timeout_ms: int = 0, + marked: bool = True, + bind: Callable[[Any], bool] | None = None, +) -> Any: + """Initialize a session; an explicit owner binds before initialization can wait. + + A successful synchronous bind transfers cleanup, including when it returns False + to refuse initialization. A bind that raises leaves cleanup with this connector. + """ import asyncpg with _opening_scope() as gate: _require_dsn() conn = None + owned = False try: conn = await asyncpg.connect( settings.pg_dsn, @@ -654,13 +664,18 @@ async def _connect_dedicated(*, statement_timeout_ms: int = 0, marked: bool = Tr **(marker_settings() if marked else {}), }, ) + if bind is not None: + usable = bind(conn) + owned = True + if not usable: + raise retirement.ProcessStopping("Postgres connection owner is closed") _check_unsealed(gate) await _check_deployment(conn) await init_connection(conn) _check_unsealed(gate) return conn except BaseException as error: - if conn is not None: + if conn is not None and not owned: with suppress(BaseException): _terminate(conn) if isinstance(error, Exception): diff --git a/server/router/app.py b/server/router/app.py index 52c9d3d157..87cc8d775e 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -14,6 +14,8 @@ - `POST /cells/v1/operator/cells//limit` and `.../credential`: the operator's changes to a cell's row (`loom cell limit`, `loom cell credential`), with the router's operator credential; +- `POST /cells/v1/operator/cells//place`: `loom cell place` places an org with the + operator credential, verifying the requested cell's deployment id and fingerprint; - `POST /cells/v1/operator/cells//register`: `loom cell prepare` registers a cell, once the probe listener (the first cell's at LOOM_ROUTER_UPSTREAM_PROBE_URL, a later cell's at its `probe_url`) answers its deployment id, prepare's nonces and the bucket and @@ -21,6 +23,8 @@ - `GET /cells/v1/operator/cells/`: the cell's row as attach reads it; - `POST /cells/v1/operator/cells//attach/begin`, `.../attach/place`, `.../attach/enforced` and `.../attach/serving`: the first cell's attach, in that order; +- `POST /cells/v1/operator/cells//configuration`: reserve or narrow the cell's + configuration after store proof, with the claimed generation and exact observed pair; - `DELETE /cells/v1/operator/cells/`: `loom cell remove` removes the row of a cell that is `attaching` with no placement; - `GET /livez` and `GET /readyz` (the directory answers, read at most once a second). @@ -28,9 +32,9 @@ Every refusal of a placement writes nothing. A malformed placement body answers 400; the directory's refusals answer 503, so the cell's request answers 503 and its retry is safe, except a new org longer than `cell.ORG_MAX_BYTES`, which answers 400 `ORG_TOO_LONG` and no -retry places; a credential that names no cell, or not this one, answers 401. An operator -route answers a cell the directory does not hold 404 with the code `NO_SUCH_CELL`, so a 404 -without it is a router of an earlier build. +retry places; a credential that names no cell, or not this one, answers 401. Operator +placement refusals answer 409; the other operator routes answer a cell the directory does +not hold 404 with `NO_SUCH_CELL`, so a 404 without it is a router of an earlier build. """ from __future__ import annotations @@ -49,6 +53,7 @@ from collections import OrderedDict from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager +from dataclasses import replace from typing import Any, NamedTuple, Protocol from urllib.parse import parse_qs from uuid import UUID @@ -62,7 +67,7 @@ from server.core import cell as cell_state from server.core.cell_probe import PATH as PROBE_PATH -from server.router import directory, routing +from server.router import directory, fingerprints, routing from server.router.limits import ( ATTACH_BATCH, ATTACH_BEGIN_MAX_BYTES, @@ -302,7 +307,7 @@ def refusal(locked: directory.Cell) -> str: return "" try: - winner, deployment_id = await directory.place(org, name, refusal) + winner, deployment_id = await directory.place(org, name, refusal, actor=f"cell:{name}") except directory.OrgTooLong as e: return _refused(400, e.reason, code="ORG_TOO_LONG") except (directory.PlacementRefused, directory.DirectoryUnavailable) as e: @@ -315,6 +320,41 @@ def refusal(locked: directory.Cell) -> str: log.info("placement of %s: %s (asked by %s)", org, winner, name) return JSONResponse({"cell": winner, "deployment_id": deployment_id}) + async def operator_place(self, request: Request) -> JSONResponse: + """Place with the operator credential. The CLI checks its local Postgres mode; + this route trusts that check and verifies the directory's locked identity.""" + body = await self._operator_body(request, MAX_BODY_BYTES) + if isinstance(body, JSONResponse): + return body + org = cell_state.named_org(body.get("org")) + deployment_id = _text(body, "deployment_id", needed=True) + if ( + set(body) != {"org", "deployment_id", "fingerprint"} + or org is None + or deployment_id is None + ): + return _refused(400, "a placement names org, deployment_id and fingerprint") + if reason := cell_state.org_name_refusal(org): + return _refused(400, reason, code="ORG_TOO_LONG") + name = request.path_params["cell"] + + def verify(locked: directory.Cell) -> str: + if locked.deployment_id != deployment_id: + return "the calling process names another deployment id" + if not fingerprint_matches(locked, body["fingerprint"]): + return f"the calling process's configuration matches no value of {name!r}" + return "" + + try: + winner, deployment = await directory.place( + org, name, placement_refusal, actor="operator", verify=verify + ) + except directory.PlacementRefused as e: + return _refused(409, e.reason) + except directory.DirectoryUnavailable as e: + return _refused(503, e.reason, code="BACKEND_UNAVAILABLE") + return JSONResponse({"org": org, "cell": winner, "deployment_id": deployment}) + async def cell_state(self, request: Request) -> JSONResponse: """`GET /cells/v1/cells/`: the first cell's state, for its own credential only, from this replica's last read of the row (`_state_row`). A first cell in @@ -686,6 +726,135 @@ async def claim(self, request: Request) -> JSONResponse: lambda: directory.claim(request.path_params["cell"], expected), _attach_answer ) + async def addresses(self, request: Request) -> JSONResponse: + if request.method == "GET": + if refused := self._operator(request) or _malformed_cell_refusal(request): + return refused + found = await _read_attaching(request.path_params["cell"]) + if isinstance(found, JSONResponse): + return found + return JSONResponse( + { + **_attach_answer(found), + "router_upstream_url": _cell_address(settings.router_upstream_url) or "", + "router_upstream_probe_url": _cell_address(settings.router_upstream_probe_url) + or "", + } + ) + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if set(body) != {"probe_url", "upstream_url", "nonces"} or not _nonces_shaped( + body.get("nonces") + ): + return _refused( + 400, "address refresh requires probe_url, upstream_url and store nonces" + ) + probe_text = _text(body, "probe_url", needed=True) + upstream_text = _text(body, "upstream_url") + probe_url = _cell_address(probe_text) if probe_text is not None else None + upstream_url = _cell_address(upstream_text) if upstream_text else "" + if not probe_url or upstream_text is None or upstream_url is None: + return _refused(400, "address refresh requires normalized HTTP(S) host-only addresses") + found = await _read_attaching(request.path_params["cell"]) + if isinstance(found, JSONResponse): + return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) + proposed = replace(found, probe_url=probe_url, upstream_url=upstream_url) + + def configured() -> bool: + return not found.row.is_first or (probe_url, upstream_url) == ( + _cell_address(settings.router_upstream_probe_url), + _cell_address(settings.router_upstream_url), + ) + + if not configured(): + return _refused( + 409, "proposed addresses differ from this router's configured upstreams" + ) + if not found.row.is_first and upstream_url != found.upstream_url: + return _refused(409, "a later cell probe refresh cannot change its upstream address") + if refused := await _probe( + proposed, body["nonces"], self._probe_transport, writer="update" + ): + return refused + if found.row.is_first: + if refused := await _upstream_check(proposed, self._probe_transport): + return refused + response = await _answered( + lambda: directory.update_addresses( + found.row.cell, probe_url, upstream_url, expected=expected, configured=configured + ), + _attach_answer, + ) + if response.status_code in (200, 503): + self._table_at = None + return response + + async def configuration_placements(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + orgs = body.get("orgs") + if ( + set(body) != {"orgs", "keys_fp"} + or _text(body, "keys_fp", needed=True) is None + or not isinstance(orgs, list) + or any(cell_state.named_org(org) is None for org in orgs) + ): + return _refused( + 400, "placement proof requires keys_fp and an explicit list of configured org names" + ) + if any(cell_state.org_name_refusal(org) for org in orgs): + return _refused(400, "a configured org name is too long", code="ORG_TOO_LONG") + return await _answered( + lambda: directory.configuration_placements( + request.path_params["cell"], orgs, expected=expected, keys_fp=body["keys_fp"] + ), + lambda held: {"cell": request.path_params["cell"], "orgs": held}, + ) + + async def configuration(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if set(body) != {"action", "target_fp", "keys_fp", "nonces"}: + return _refused(400, "configuration requires action, target_fp, keys_fp and nonces") + action, target, keys_fp = body["action"], body["target_fp"], body["keys_fp"] + if action not in ("config", "finish", "revert") or not fingerprints.readable(target): + return _refused(400, "configuration requires a supported action and readable target_fp") + if _text(body, "keys_fp", needed=True) is None: + return _refused(400, f"keys_fp is text of 1 to {_TEXT_MAX} characters, no NUL") + if not _nonces_shaped(body["nonces"]): + return _refused(400, _NONCES_REFUSAL) + name = request.path_params["cell"] + found = await _read_attaching(name) + if isinstance(found, JSONResponse): + return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) + if refused := await self._stores_check(found, body["nonces"]): + return refused + response = await _answered( + lambda: directory.update_configuration( + name, target, action, keys_fp=keys_fp, refusal=placement_refusal, expected=expected + ), + _attach_answer, + ) + if response.status_code in (200, 503): + self._table_at = None + return response + async def attach_begin(self, request: Request) -> JSONResponse: """`attach/begin`: once the probe listener echoes attach's nonces from the stores the row names and the upstream reports the row's deployment id, write when attach @@ -1460,6 +1629,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: return Starlette( routes=[ Route("/cells/v1/placements", router.place, methods=["POST"]), + Route("/cells/v1/operator/cells/{cell}/place", router.operator_place, methods=["POST"]), Route("/cells/v1/cells/{cell}", router.cell_state, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/limit", router.set_limit, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}/register", router.register, methods=["POST"]), @@ -1470,6 +1640,21 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: ), Route("/cells/v1/operator/cells/{cell}", router.operator_cell, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/claim", router.claim, methods=["POST"]), + Route( + "/cells/v1/operator/cells/{cell}/addresses", + router.addresses, + methods=["GET", "POST"], + ), + Route( + "/cells/v1/operator/cells/{cell}/configuration/placements", + router.configuration_placements, + methods=["POST"], + ), + Route( + "/cells/v1/operator/cells/{cell}/configuration", + router.configuration, + methods=["POST"], + ), Route("/cells/v1/operator/cells/{cell}", router.remove, methods=["DELETE"]), Route( "/cells/v1/operator/cells/{cell}/attach/begin", diff --git a/server/router/directory.py b/server/router/directory.py index 3e438cf6d9..aa7ac03328 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -8,11 +8,13 @@ (`attach_place`) both take before they insert, so a cell's count and quota are judged one placement at a time; a live placement reads the winner again once it holds the lock, so an org a batch placed meanwhile is answered as placed; -- a transaction-scoped lock on the org (`_PLACE_LOCK`) between live placements, so two cells - placing one org at once agree on the winner, and a retry of the same call answers the - same; a batch takes none, since no other cell exists while it runs; -- the placements' primary key, as a backstop: every insert is `ON CONFLICT DO NOTHING`, and - only a row it returns is counted and audited. +- transaction-scoped org locks (`_PLACE_LOCK`) before cell locks for live placements and + configuration updates. Configuration takes distinct numeric hash keys in order; the + first attach batch needs none while no other cell exists; +- the placements' primary key, as a backstop. Live placement and first attach use + `ON CONFLICT DO NOTHING`; configuration inserts after its locked winner read. Only + inserted rows are counted. Every successful call writes an audit row before committing, + so a duplicate winner waits for synchronous durability too. """ from __future__ import annotations @@ -31,7 +33,7 @@ from server.core import cell as cell_state from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR -from server.router.limits import MAX_BODY_BYTES +from server.router import fingerprints from server.settings import settings # The directory's own migration lock: never the memory authority's. @@ -62,6 +64,10 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) +# New placement: BEGIN, org lock, winner, cell lock, winner, insert, counter, winner, +# audit, COMMIT. The audit also gives duplicate winners a synchronous durability fence. +PLACE_STATEMENTS = 10 +PLACE_MAX_S = _bound_s(PLACE_STATEMENTS) # not a setting: derived # The longest limit/credential change has ten statements: begin, directory lock, row, # other row, WAL, credential check, update, audit, read-back and commit. OPERATOR_CHANGE_MAX_S = _bound_s(10) # not a setting: derived from the bound above @@ -167,11 +173,20 @@ async def _init(conn: Any) -> None: typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog" ) except BaseException as original: - # Only builtin codecs run here: this raw session has never been borrowed or - # held command locks. Quarantine before the driver can lose its ownership. + # Failed initialization cannot admit this raw session. A local abort alone + # does not acknowledge backend exit, so finish public close while still owned. + task = asyncio.current_task() + unknown = task is None or bool(task.cancelling()) + if not unknown: + try: + await conn.close(timeout=STATEMENT_TIMEOUT_S) + except BaseException: # noqa: BLE001 — close must preserve the first init failure. + unknown = True try: conn.terminate() - except BaseException: # noqa: BLE001 — disposal must preserve the first init failure. + except BaseException: # noqa: BLE001 — abort cannot replace the first init failure. + unknown = True + if unknown: try: original.add_note("directory initialization session closure is unknown") except BaseException: # noqa: BLE001 — diagnostics must preserve the first failure. @@ -674,44 +689,53 @@ async def first_cell_named(cell_name: str) -> Cell | None: return _cell(row) if row else None -async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: - """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) - that holds it. An org placed already answers its cell, whatever the cell's state. - Otherwise the cell's row is locked, the winner read again (an attach batch holds the row - lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it - waits for the placement and the quota holds across router replicas; a refused placement - raises PlacementRefused and writes nothing, and a new org name longer than - `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as - any placed org does).""" +async def place( + org: str, + cell_name: str, + refusal: Callable[[Cell], str], + *, + actor: str, + verify: Callable[[Cell], str] | None = None, +) -> tuple[str, str]: + """The permanent winner of `org`, with every successful call audited and committed + synchronously. `verify` judges the requested cell's locked identity even for a winner + already there; `refusal` and quota judge new placements only. A winner elsewhere needs + neither check. The org lock and cell row lock serialize live and attach placements.""" async with _reading(), _connection() as conn, conn.transaction(): - # Placements of one org run one at a time, whichever cell asks: the later one - # reads the earlier one's winner. await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) row = await conn.fetchrow(_WINNER, org) - if row is None: - _refuse_long([org]) + if row is None or (verify is not None and row["cell"] == cell_name): + if row is None: + _refuse_long([org]) locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) if locked is None: raise PlacementRefused(f"cell {cell_name!r} is not in the directory") - # An attach batch of this cell, which holds the row lock and not the org's, may - # have placed the org since the first read: it is answered as placed. + candidate = _cell(locked) + # An attach batch holds this row's lock rather than the org's. row = await conn.fetchrow(_WINNER, org) + if (row is None or row["cell"] == cell_name) and verify is not None: + if reason := verify(candidate): + raise PlacementRefused(reason) if row is None: - await _place_one(conn, org, cell_name, _cell(locked), refusal) + await _place_one(conn, org, cell_name, candidate, refusal) row = await conn.fetchrow(_WINNER, org) - return str(row["cell"]), str(row["deployment_id"]) + winner = str(row["cell"]), str(row["deployment_id"]) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + actor, + {"org": org, "cell": winner[0]}, + ) + return winner async def _place_one( conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] ) -> None: - """Judge the locked row (`refusal`, then its quota) and insert the placement; its count - and audit row only when the insert wrote it.""" + """Judge the locked row and insert the placement, counted only when written.""" if reason := refusal(candidate): raise PlacementRefused(reason) - quota = candidate.placements_max - if quota is not None and candidate.placed >= quota: - raise PlacementRefused(f"cell {cell_name!r} holds its quota of orgs") + if reason := _quota_refusal(candidate, 1): + raise PlacementRefused(reason) inserted = await conn.fetchval( "INSERT INTO loom_router.placements (org, cell) VALUES ($1, $2) " "ON CONFLICT DO NOTHING RETURNING org", @@ -723,11 +747,12 @@ async def _place_one( await conn.execute( "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name ) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", - f"cell:{cell_name}", - {"org": org, "cell": cell_name}, - ) + + +def _quota_refusal(row: Cell, additional: int) -> str: + if row.placements_max is not None and row.placed + additional > row.placements_max: + return f"cell {row.cell!r} holds its quota of orgs" + return "" def _refuse_long(orgs: Iterable[str]) -> None: @@ -821,50 +846,6 @@ def attach_refusal( return "" -def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> str: - """Why a posted fingerprint is not this row's cell ('' when it is): its stores and - hosts, the servers' own values, equal the row's.""" - fields = value["fields"] - row = found.row - expected = { - "deployment_id": row.deployment_id, - "bucket": found.bucket, - "bucket_prefix": found.bucket_prefix, - "public_host": row.host, - } - differ = [k for k, want in expected.items() if fields.get(k, cell_state.DEFAULTS[k]) != want] - if differ: - return f"{name}'s {', '.join(differ)} differ from cell {row.cell!r}'s registration" - issuer_host = fields.get("oauth_issuer_host", cell_state.DEFAULTS["oauth_issuer_host"]) - if issuer_host != row.host: - return ( - f"{name}'s oauth_issuer_host is not cell {row.cell!r}'s host {row.host!r}: the " - "router attaches no issuer host other than the cell's own" - ) - return "" - - -# An org as a placement body carries it at most: ORG_MAX_BYTES bytes, each escaped in JSON -# to at most six (`\u00XX`). -_PLACEMENT_ORG_MAX = 6 * cell_state.ORG_MAX_BYTES # not a setting: derived - - -def _placement_refusal(cell: str, name: str, value: dict[str, Any]) -> str: - """Why a fingerprint is too large for the cell's live placements ('' when it is not): a - placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's - MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" - # The default encoding is the longest: spaces after separators, non-ASCII escaped. - size = len(json.dumps({"org": "", "cell": cell, "fingerprint": value})) + _PLACEMENT_ORG_MAX - if size <= MAX_BODY_BYTES: - return "" - sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) - largest = ", ".join(k for _, k in sizes[:3]) - return ( - f"{name} makes a placement body of up to {size} bytes, past the router's " - f"{MAX_BODY_BYTES}: its largest fields are {largest}" - ) - - def _as_json(*values: Any) -> str: """Values in one canonical JSON text, which tells `false`, `0` and `0.0` apart.""" return json.dumps(values, sort_keys=True) @@ -876,13 +857,17 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: if reason := attach_refusal(found, begin.deployment_id, begun=False): return reason row = found.row - for name, value in (("current_fp", begin.current_fp), ("next_fp", begin.next_fp)): - if value is None: - continue - if reason := _fingerprint_refusal(found, name, value) or _placement_refusal( - row.cell, name, value - ): - return reason + registered = { + "cell": row.cell, + "deployment_id": row.deployment_id, + "bucket": found.bucket, + "bucket_prefix": found.bucket_prefix, + "host": row.host, + } + if reason := fingerprints.registration_pair_refusal( + registered, begin.current_fp, begin.next_fp + ): + return reason # Compared as JSON, not as Python values, where `false == 0` and `1 == 1.0`: a rerun # writes the fingerprints again only when they are the row's to the type. if row.current_fp is not None and _as_json(row.current_fp, row.next_fp) != _as_json( @@ -1120,6 +1105,227 @@ async def change_locked(conn: Any, _found: Attaching) -> Attaching: return await _attach_change(cell_name, expected, change_locked, claim=True) +async def update_addresses( + cell_name: str, + probe_url: str, + upstream_url: str, + *, + expected: Control, + configured: Callable[[], bool], +) -> Attaching: + """Commit only the addresses whose stores were just proved outside this transaction.""" + + async def change(conn: Any, found: Attaching) -> Attaching: + if not configured(): + raise OperatorRefused( + "the contacted router's configured addresses changed during proof" + ) + if not found.row.is_first and upstream_url != found.upstream_url: + raise OperatorRefused("a later cell probe refresh cannot change its upstream address") + await conn.execute( + ( + "UPDATE loom_router.cells SET probe_url=$2, upstream_url=$3, updated_at=now() " + "WHERE cell=$1" + ), + cell_name, + probe_url, + upstream_url, + ) + await _audit( + conn, + "addresses", + {"cell": cell_name, "probe_url": probe_url, "upstream_url": upstream_url}, + ) + row = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell=$1", cell_name) + return _attaching(row, found.other) + + return await _attach_change(cell_name, expected, change) + + +async def configuration_placements( + cell_name: str, orgs: list[str], *, expected: Control, keys_fp: str +) -> list[str]: + """Acknowledge actual immutable winners before the cell repairs local tenancy.""" + _refuse_long(orgs) + + async def read(conn: Any, found: Attaching) -> list[str]: + if ( + await conn.fetchval("SELECT keys_fp FROM loom_router.cells WHERE cell=$1", cell_name) + != keys_fp + ): + raise OperatorRefused( + "cell root-secret fingerprint changed; configuration update refuses" + ) + accepted = [ + value for value in (found.row.current_fp, found.row.next_fp) if value is not None + ] + if not accepted or any(not fingerprints.readable(value) for value in accepted): + raise OperatorRefused( + "configured placement proof needs readable directory reservations" + ) + configured = { + org for value in accepted for org in value["fields"].get("configured_orgs", []) + } + if not set(orgs) <= configured: + raise OperatorRefused("placement proof names an org outside the reserved configuration") + rows = await conn.fetch( + "SELECT org, cell FROM loom_router.placements WHERE org=ANY($1::text[]) ORDER BY org", + sorted(set(orgs)), + ) + for row in rows: + if row["cell"] != cell_name: + raise OperatorRefused(f"org {row['org']!r} is placed in cell {row['cell']!r}") + await _audit( + conn, + "configuration_placements", + {"cell": cell_name, "named": len(set(orgs)), "held": len(rows)}, + ) + return [str(row["org"]) for row in rows] + + return await _attach_change(cell_name, expected, read) + + +async def update_configuration( + cell_name: str, + target: dict[str, Any], + action: str, + *, + keys_fp: str, + refusal: Callable[[Cell], str], + expected: Control, +) -> Attaching: + """Reserve acceptance and configured placements in one bounded directory commit.""" + try: + current, pending = fingerprints.configuration_pair( + expected.values.get("current_fp"), expected.values.get("next_fp"), target, action + ) + except fingerprints.ConfigurationRefused as error: + raise OperatorRefused(error.reason) from None + accepted = [value for value in (current, pending) if value is not None] + desired_orgs = {org for value in accepted for org in value["fields"].get("configured_orgs", [])} + prior_orgs = { + org + for value in (expected.values["current_fp"], expected.values["next_fp"]) + if value is not None + for org in value["fields"].get("configured_orgs", []) + } + orgs = sorted(desired_orgs | prior_orgs) + _refuse_long(orgs) + + async def locked(conn: Any) -> Attaching: + # Numeric hash order matches live placement locks, including hash collisions. + await conn.execute( + "WITH keys AS MATERIALIZED (SELECT DISTINCT hashtext(org) AS key " + "FROM unnest($2::text[]) AS org ORDER BY key) " + "SELECT pg_advisory_xact_lock($1, key) FROM keys ORDER BY key", + _PLACE_LOCK, + orgs, + ) + await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) + raw = await conn.fetchrow( + f"SELECT {_CELL_COLUMNS}, {', '.join(_ATTACH_EXTRA)}, keys_fp " + "FROM loom_router.cells WHERE cell=$1 FOR UPDATE", + cell_name, + ) + if raw is None: + raise NoSuchCell(f"cell {cell_name!r} is not in the directory") + row = dict(raw) + recorded_keys = row.pop("keys_fp") + others = await conn.fetch( + "SELECT cell, current_fp, next_fp FROM loom_router.cells WHERE cell<>$1 ORDER BY cell", + cell_name, + ) + found = _attaching(row, str(others[0]["cell"]) if others else None) + if reason := control_refusal(found, expected): + raise OperatorRefused(reason) + if recorded_keys != keys_fp: + raise OperatorRefused( + "cell root-secret fingerprint changed; configuration update refuses" + ) + registered = {name: getattr(found, name) for name in ("bucket", "bucket_prefix")} + registered.update( + cell=cell_name, deployment_id=found.row.deployment_id, host=found.row.host + ) + if reason := fingerprints.registration_pair_refusal(registered, current, pending): + raise OperatorRefused(reason) + for other in others: + reserved = [ + other[name] for name in ("current_fp", "next_fp") if other[name] is not None + ] + if reason := fingerprints.reservation_refusal(accepted, reserved): + raise OperatorRefused(f"cell {other['cell']!r}: {reason}") + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + placed = await conn.fetch( + "SELECT org, cell FROM loom_router.placements WHERE org=ANY($1::text[]) ORDER BY org", + orgs, + ) + elsewhere = next((row for row in placed if row["cell"] != cell_name), None) + if elsewhere is not None: + raise OperatorRefused( + f"org {elsewhere['org']!r} is placed in cell {elsewhere['cell']!r}" + ) + new = sorted(set(orgs) - {row["org"] for row in placed}) + if not found.row.is_first and found.row.state != "serving": + if desired_orgs != prior_orgs: + raise OperatorRefused("a later cell must be serving before configured orgs change") + if set(new) & prior_orgs: + if not (found.row.is_first and found.row.state == "attaching"): + raise OperatorRefused( + "configured org placement is missing; " + "restore the directory metadata before updating" + ) + # Original first-cell orgs retain attach's placement and quota protocol. + new = sorted(set(new) - prior_orgs) + if new: + if reason := refusal(found.row): + raise OperatorRefused(reason) + if reason := _quota_refusal(found.row, len(new)): + raise OperatorRefused(reason) + return await _configuration_commit(conn, found, current, pending, new, action) + + return await _control_transaction(locked, seconds=CONTROL_MAX_S) + + +async def _configuration_commit( + conn: Any, + found: Attaching, + current: dict[str, Any], + pending: dict[str, Any] | None, + new_orgs: list[str], + action: str, +) -> Attaching: + accepted_issuers = sorted( + { + issuer + for value in (current, pending) + if value is not None + for issuer in value["fields"].get("bearer_issuers", []) + if issuer + } + ) + written = await conn.fetchrow( + "WITH inserted AS (INSERT INTO loom_router.placements (org, cell) " + "SELECT org, $1 FROM unnest($2::text[]) AS org RETURNING org), " + "placed_audit AS (INSERT INTO loom_router.audit (actor, action, detail) " + "SELECT 'operator', 'place', jsonb_build_object('org', org, 'cell', $1::text) " + "FROM inserted), " + "changed AS (UPDATE loom_router.cells SET current_fp=$3, next_fp=$4, " + "issuers=$5, placed=placed+(SELECT count(*) FROM inserted), updated_at=now() " + f"WHERE cell=$1 RETURNING {_CELL_COLUMNS}, {', '.join(_ATTACH_EXTRA)}), " + "changed_audit AS (INSERT INTO loom_router.audit (actor, action, detail) " + "SELECT 'operator', 'configuration_' || $6::text, " + "jsonb_build_object('cell', cell, 'placed', (SELECT count(*) FROM inserted)) FROM changed) " + "SELECT * FROM changed", + found.row.cell, + new_orgs, + current, + pending, + accepted_issuers if found.row.state == "serving" else [], + action, + ) + return _attaching(written, found.other) + + async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: await conn.execute( "INSERT INTO loom_router.audit (actor, action, detail) VALUES ('operator', $1, $2)", diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py new file mode 100644 index 0000000000..0a818f5a3b --- /dev/null +++ b/server/router/fingerprints.py @@ -0,0 +1,212 @@ +"""Pure checks on the fingerprint reservations shared by cell controllers.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from server.core import cell, org_patterns +from server.router.limits import MAX_BODY_BYTES + + +def readable(value: Any) -> bool: + """Known fields have exact types; unknown JSON fields remain reserved and intact.""" + if not isinstance(value, dict) or set(value) != {"v", "fields"}: + return False + if type(value["v"]) is not int or value["v"] != cell.FINGERPRINT_VERSION: + return False + fields = value["fields"] + if not isinstance(fields, dict): + return False + try: + # Reject cycles, non-JSON values and nonfinite floats before walking nested fields. + json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + return False + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + try: + if "\0" in item: + return False + item.encode() + except UnicodeError: + return False + elif isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + return False + pending.extend(item.keys()) + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + elif item is not None and type(item) not in (bool, int, float): + return False + for name, default in cell.DEFAULTS.items(): + item = fields.get(name, default) + if type(item) is not type(default): + return False + if isinstance(item, list) and any(not isinstance(part, str) for part in item): + return False + for name in ("configured_orgs", "end_user_orgs"): + for org in fields.get(name, []): + if cell.named_org(org) is None or cell.org_name_refusal(org): + return False + return True + + +def registration_refusal(value: Any, registered: Mapping[str, Any], name: str = "candidate") -> str: + """A fingerprint accepts only the registered stores and host.""" + if not readable(value): + return f"{name}: unreadable fingerprint; change fingerprints with `loom cell update`" + expected = { + "deployment_id": registered["deployment_id"], + "bucket": registered["bucket"], + "bucket_prefix": registered["bucket_prefix"], + "public_host": registered["host"], + } + fields = value["fields"] + changed = [ + name for name, want in expected.items() if fields.get(name, cell.DEFAULTS[name]) != want + ] + if changed: + return ( + f"{name}'s {', '.join(changed)} differ from cell {registered['cell']!r}'s registration" + ) + if fields.get("oauth_issuer_host", cell.DEFAULTS["oauth_issuer_host"]) != registered["host"]: + return ( + f"{name}'s oauth_issuer_host differs from the registered host: the router " + "attaches no issuer host other than the cell's own" + ) + return "" + + +def registration_pair_refusal( + registered: Mapping[str, Any], current: dict[str, Any], pending: dict[str, Any] | None +) -> str: + """Both reserved values must name the registered stores and fit live placement.""" + for name, value in (("current_fp", current), ("next_fp", pending)): + if value is not None: + if reason := registration_refusal(value, registered, name): + return reason + if reason := placement_refusal(registered["cell"], name, value): + return reason + return "" + + +def _pair_refusal(a: dict[str, Any], b: dict[str, Any]) -> str: + if set(a.get("bearer_issuers", [])) & set(b.get("bearer_issuers", [])): + return "shared bearer issuer reservation" + oidc_a = (a.get("oidc_issuer", "").rstrip("/"), a.get("oidc_client_id", "")) + oidc_b = (b.get("oidc_issuer", "").rstrip("/"), b.get("oidc_client_id", "")) + if all(oidc_a) and oidc_a == oidc_b: + return "shared OIDC issuer and client reservation" + patterns_a, patterns_b = a.get("end_user_orgs", []), b.get("end_user_orgs", []) + if any(org_patterns.overlap(x, y) for x in patterns_a for y in patterns_b): + return "overlapping end-user org pattern reservation" + orgs_a, orgs_b = a.get("configured_orgs", []), b.get("configured_orgs", []) + if ( + set(orgs_a) & set(orgs_b) + or any(org_patterns.matches(pattern, org) for pattern in patterns_a for org in orgs_b) + or any(org_patterns.matches(pattern, org) for pattern in patterns_b for org in orgs_a) + ): + return "overlapping configured org reservation" + return "" + + +def reservation_refusal(accepted: Sequence[dict[str, Any]], other: Sequence[dict[str, Any]]) -> str: + """Compare every accepted value, including next, with another cell's reservation. + + Registration identity, placement winners and transaction locks are the caller's + checks. Empty `other` means registration has not reserved a fingerprint yet. + """ + if not accepted or any(not readable(value) for value in (*accepted, *other)): + return "unreadable fingerprint reservation" + for a in accepted: + for b in other: + if reason := _pair_refusal(a["fields"], b["fields"]): + return reason + return "" + + +class ConfigurationRefused(ValueError): + """A configuration action refused for a fixed, caller-facing reason.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +def configuration_pair( + current: dict[str, Any] | None, + pending: dict[str, Any] | None, + target: dict[str, Any], + action: str, +) -> tuple[dict[str, Any], dict[str, Any] | None]: + """Select the reserved pair without guessing the direction of an interrupted update.""" + if any(not readable(value) for value in (current, target)) or ( + pending is not None and not readable(pending) + ): + raise ConfigurationRefused("configuration update needs readable fingerprints") + assert current is not None + if action not in ("config", "finish", "revert"): + raise ConfigurationRefused("configuration action must be config, finish or revert") + + def same(a: dict[str, Any], b: dict[str, Any]) -> bool: + return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + + if action == "config": + if pending is not None and not same(pending, target): + raise ConfigurationRefused( + "another configuration is pending; finish or revert it first" + ) + if pending is not None: + return current, pending + return current, None if same(current, target) else target + selected = pending if action == "finish" and pending is not None else current + if not same(selected, target): + raise ConfigurationRefused( + "configuration target differs from the selected reserved fingerprint" + ) + return target, None + + +# An org as a placement body carries it at most: ORG_MAX_BYTES bytes, each escaped in JSON +# to at most six (`\u00XX`). +_PLACEMENT_ORG_MAX = 6 * cell.ORG_MAX_BYTES # not a setting: derived + + +def placement_refusal(cell_name: str, name: str, value: dict[str, Any]) -> str: + """Why a fingerprint cannot fit both live and operator placement bodies. + + Reserve the longest escaped org in each envelope and name only the largest fields. + """ + # Native callers materialize known defaults; retain unknown stored fields too. + value = {**value, "fields": {**cell.DEFAULTS, **value["fields"]}} + # The default encoding is the longest: spaces after separators, non-ASCII escaped. + size = ( + max( + len(json.dumps({"org": "", "cell": cell_name, "fingerprint": value})), + len( + json.dumps( + { + "org": "", + "deployment_id": value["fields"].get( + "deployment_id", cell.DEFAULTS["deployment_id"] + ), + "fingerprint": value, + } + ) + ), + ) + + _PLACEMENT_ORG_MAX + ) + if size <= MAX_BODY_BYTES: + return "" + sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) + largest = ", ".join(k for _, k in sizes[:3]) + return ( + f"{name} makes a placement body of up to {size} bytes, past the router's " + f"{MAX_BODY_BYTES}: its largest fields are {largest}" + ) diff --git a/server/settings/catalog.py b/server/settings/catalog.py index e841194021..8c2f15170d 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -28,6 +28,7 @@ import idna from pydantic import AfterValidator, BaseModel, Field, PrivateAttr, field_validator, model_validator +from server.core import org_patterns from server.core.document_formats import route_key from server.core.embedding_space import inferred_provider from server.core.numbers import parse_budget_value @@ -97,17 +98,6 @@ def _public_numbers(pem: str) -> object: return None -def _overlap(a: str, b: str) -> bool: - """Whether two org patterns (a name, or `prefix*`) can match one org.""" - if a.endswith("*") and b.endswith("*"): - return a[:-1].startswith(b[:-1]) or b[:-1].startswith(a[:-1]) - if a.endswith("*"): - return b.startswith(a[:-1]) - if b.endswith("*"): - return a.startswith(b[:-1]) - return a == b - - @dataclass(frozen=True) class EndUserIssuer: """One end-user issuer: the orgs its tokens may name and where its keys are.""" @@ -148,7 +138,7 @@ def parse(cls, issuer: str, spec: object) -> EndUserIssuer: def signs_for(self, org: str) -> bool: """Whether this issuer's tokens may name `org`: listed, or under a `prefix*`.""" - return any(org.startswith(o[:-1]) if o.endswith("*") else org == o for o in self.orgs) + return any(org_patterns.matches(o, org) for o in self.orgs) class AuthSettings(BaseModel): @@ -519,7 +509,9 @@ def _end_user_issuers_stand_apart(self) -> AuthSettings: raise ValueError(f"end-user issuer {issuer!r}: keys {same} are federated keys") # `user:` names one person per org, so one issuer alone signs for an org. for (a, spec_a), (b, spec_b) in itertools.combinations(issuers.items(), 2): - shared = [(x, y) for x in spec_a.orgs for y in spec_b.orgs if _overlap(x, y)] + shared = [ + (x, y) for x in spec_a.orgs for y in spec_b.orgs if org_patterns.overlap(x, y) + ] if shared: raise ValueError(f"end-user issuers {a!r} and {b!r} both sign for {shared[0]}") return self diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 77b3e297ce..27cecae66b 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -29,9 +29,19 @@ async def test_attach_owner_releases_backend_locks_before_return( database, monkeypatch, owner, phase, interruption ): + import os + import asyncpg + from server.settings import settings + from tests.integration._pg_database import with_database + conn, observer = database + dsn = with_database( + os.environ["LOOM_PG_DSN"], await observer.fetchval("SELECT current_database()") + ) + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(pg, "_expected", None) await conn.execute( "CREATE SCHEMA loom; CREATE TABLE loom.tenancy_mode(mode text, since timestamptz); INSERT INTO loom.tenancy_mode VALUES('recording', now())" ) @@ -72,7 +82,7 @@ async def ensure(conn, table, **kwargs): conn, table, statements, kwargs["lock_timeout"], 1, warn=False ) - async def connect(**_): + async def connect(*args, **kwargs): return conn monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) @@ -82,7 +92,7 @@ async def connect(**_): monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) monkeypatch.setattr(cell_control, "OPEN_S", 0.6) - monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(asyncpg, "connect", connect) await apg.take_lock(conn, "cleanup test") if owner == "trigger": work = apg._ensure_one(_owner(conn), "probe", 100, apg.Wait(0)) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index a647c2b9e5..b20f8c3624 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -389,10 +389,10 @@ async def test_native_public_close_fatal_stays_inside_owner(native_database, lat settings.cell = 'first' pg._expected = None original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('close interrupted') -real_connect, real_close = pg._connect_dedicated, asyncpg.Connection.close +real_connect, real_close = asyncpg.connect, asyncpg.Connection.close backend = {} -async def connect(**kwargs): - conn = await real_connect(**kwargs) +async def connect(*args, **kwargs): + conn = await real_connect(*args, **kwargs) backend['pid'] = conn.get_server_pid() if late == 'True': try: await asyncio.Future() @@ -407,11 +407,11 @@ async def close(self, **kwargs): async def body(owner): await owner.conn.execute('SELECT * FROM probe FOR UPDATE') return 19 -pg._connect_dedicated = connect +asyncpg.connect = connect asyncpg.Connection.close = close if late == 'True': cell_control.OPEN_S = .5 async def main(): - observer = await asyncpg.connect(settings.pg_dsn) + observer = await real_connect(settings.pg_dsn) try: try: await cell_control.run('native close', body) except BaseException as caught: diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index 3dbf64a4ab..9d9a2d2151 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -473,7 +473,7 @@ async def wait_for_lock(key): "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) # The placement has its org lock and is queued at the cell row before claim. for _ in range(100): @@ -507,7 +507,7 @@ async def wait_for_lock(key): "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) await wait_for_lock(directory._PLACE_LOCK) await cell_control.backout(_owner(local_admin), call, original) @@ -666,7 +666,9 @@ async def call(path, body): ) # Actually attempt placement before the acknowledged close returns to cleanup. try: - await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + await directory.place( + "between-close-and-cleanup", "first", app.placement_refusal, actor="cell:first" + ) except directory.PlacementRefused as exc: placement_error = str(exc) return answer diff --git a/tests/integration/test_cell_place.py b/tests/integration/test_cell_place.py new file mode 100644 index 0000000000..40a1cbdf4a --- /dev/null +++ b/tests/integration/test_cell_place.py @@ -0,0 +1,242 @@ +"""Operator placement on two cells' databases and a real directory, without memory rows.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import os +import time + +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.router import app, directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database, own_database + +OPERATOR = "lco_" + "3" * 128 +ROUTER = httpx.URL("http://127.0.0.1:7901") + + +@pytest.fixture +async def stores(monkeypatch): + admin = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, dsn = await create_database(admin) + stack = contextlib.AsyncExitStack() + try: + stack.push_async_callback(drop_database, admin, name) + first = await stack.enter_async_context(own_database()) + second = await stack.enter_async_context(own_database()) + monkeypatch.setattr(directory, "_pool", None) + stack.push_async_callback(directory.close) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + await directory.migrate() + stack.push_async_callback(pg.close_pool) + stack.callback(pg.expect_deployment, None) + pg.expect_deployment(None) + router_app = app.build_app(app.Router()) + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kw: real_client(transport=httpx.ASGITransport(app=router_app), **kw), + ) + targets = {"first": first, "second": second} + for cell_name, cell_dsn in targets.items(): + await _initialize(cell_dsn, cell_name) + yield targets + except BaseException: # Cleanup must preserve the setup failure. + with contextlib.suppress(BaseException): + await stack.aclose() + raise + else: + await stack.aclose() + + +async def _initialize(dsn, name): + import asyncpg + + fields = cell.fingerprint(f"dep-{name}") + conn = await asyncpg.connect(dsn) + try: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + f"dep-{name}", + ) + await conn.execute( + "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", name == "first", name + ) + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6)", + name, + name == "first", + f"{name}.example.test", + hashlib.sha256(name.encode()).hexdigest(), + f"dep-{name}", + cell.as_value(fields), + ) + + +async def _local(stores, monkeypatch, name, org="acme"): + monkeypatch.setattr(settings, "cell", name) + monkeypatch.setattr(settings, "pg_dsn", stores[name]) + return await cell_place.place(argparse.Namespace(cell=name, org=org), ROUTER) + + +async def _tenancy(dsn): + import asyncpg + + conn = await asyncpg.connect(dsn) + try: + return [r["org"] for r in await conn.fetch("SELECT org FROM loom.tenancy ORDER BY org")] + finally: + await conn.close() + + +async def test_operator_place_retries_and_refuses_a_different_cell( + stores, monkeypatch, record_property +): + started = time.perf_counter() + first = await _local(stores, monkeypatch, "first") + record_property("local_first_place_ms", (time.perf_counter() - started) * 1000) + assert first == {"org": "acme", "cell": "first", "deployment_id": "dep-first"} + started = time.perf_counter() + assert await _local(stores, monkeypatch, "first") == first + record_property("local_duplicate_place_ms", (time.perf_counter() - started) * 1000) + assert await _tenancy(stores["first"]) == ["acme"] + with pytest.raises(cell_place.PlacementRefused, match="belongs to cell 'first'"): + await _local(stores, monkeypatch, "second") + assert await _tenancy(stores["second"]) == [] + async with directory._connection() as conn: + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'first'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 3 + ) + + +async def test_two_cells_racing_one_org_have_one_winner_and_count_once(stores): + async def place(name): + fields = cell.as_value(cell.fingerprint(f"dep-{name}")) + async with httpx.AsyncClient() as client: + return await client.post( + str(ROUTER.copy_with(path=f"/cells/v1/operator/cells/{name}/place")), + json={"org": "raced", "deployment_id": f"dep-{name}", "fingerprint": fields}, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + a, b = await asyncio.gather(place("first"), place("second")) + assert a.status_code == b.status_code == 200 + assert a.json() == b.json() + async with directory._connection() as conn: + assert await conn.fetchval("SELECT sum(placed) FROM loom_router.cells") == 1 + assert await conn.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + assert await _tenancy(stores["first"]) == await _tenancy(stores["second"]) == [] + + +@pytest.mark.parametrize("limit", ["closed = true", "placements_max = 0"]) +async def test_new_closed_or_full_cell_writes_neither_store(stores, monkeypatch, limit): + async with directory._connection() as conn: + await conn.execute(f"UPDATE loom_router.cells SET {limit} WHERE cell = 'first'") + with pytest.raises(cell_place.PlacementRefused, match="router refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert await directory.lookup("acme") is None + + +async def test_failed_local_insert_resumes_and_existing_winner_ignores_closed_quota( + stores, monkeypatch +): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute( + "CREATE FUNCTION loom.fail_place() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'local insertion refused'; END $$" + ) + await conn.execute( + "CREATE TRIGGER fail_place BEFORE INSERT ON loom.tenancy FOR EACH ROW EXECUTE FUNCTION loom.fail_place()" + ) + with pytest.raises(asyncpg.RaiseError, match="local insertion refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert (await directory.lookup("acme"))[0] == "first" + await conn.execute("DROP TRIGGER fail_place ON loom.tenancy") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +async def test_first_recording_refuses_and_enforced_attaching_succeeds(stores, monkeypatch): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + with pytest.raises(cell_place.PlacementRefused, match="needs enforced"): + await _local(stores, monkeypatch, "first") + assert await directory.lookup("acme") is None + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET state = 'attaching' WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +@pytest.mark.parametrize("changed", ["mode", "fingerprint"]) +async def test_changed_local_state_after_directory_call_refuses_insert( + stores, monkeypatch, changed +): + import asyncpg + + real = cell_place._directory + + async def change_after_call(*args): + await real(*args) + conn = await asyncpg.connect(stores["first"]) + try: + if changed == "mode": + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + else: + fields = cell.fingerprint("dep-first") + fields["oidc_client_id"] = "changed" + await conn.execute( + "UPDATE loom.cell_self SET current_fp = $1::jsonb", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + + monkeypatch.setattr(cell_place, "_directory", change_after_call) + with pytest.raises(cell_place.PlacementRefused): + await _local(stores, monkeypatch, "first") + assert (await directory.lookup("acme"))[0] == "first" + assert await _tenancy(stores["first"]) == [] diff --git a/tests/integration/test_cell_place_cleanup.py b/tests/integration/test_cell_place_cleanup.py new file mode 100644 index 0000000000..3bbb5acc07 --- /dev/null +++ b/tests/integration/test_cell_place_cleanup.py @@ -0,0 +1,106 @@ +"""Placement cleanup observes the actual marked backend through interrupted work.""" + +from __future__ import annotations + +import asyncio +import contextlib +import time + +import asyncpg +import pytest + +from server.cli import cell_place +from server.core import pg +from tests.integration import test_cell_place as placement + + +@pytest.fixture +async def stores(monkeypatch): + async for value in placement.stores.__wrapped__(monkeypatch): + yield value + + +async def waiting(observer, pid): + async with asyncio.timeout(3): + while True: + if await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event='PgSleep')", + pid, + ): + return + await asyncio.sleep(0.01) + + +async def assert_released(observer, pid): + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +@pytest.mark.parametrize("interrupt", ["deadline", "cancel", "second_cancel"]) +async def test_placement_owner_closes_before_cancelling_actual_rollback( + stores, monkeypatch, interrupt +): + # Only the exact ROLLBACK wire operation is delayed. The transaction context, + # sleeping query, session lock, observer and close are the installed real driver. + original_identity, execute = cell_place._identity, asyncpg.Connection.execute + known = asyncio.Event() + captured = {} + queries = [] + + async def stalled_rollback(conn, sql, *args, **kwargs): + if conn is captured.get("conn") and sql == "ROLLBACK;": + query = asyncio.create_task(execute(conn, "SELECT pg_sleep(10)")) + queries.append(query) + while not query.done(): + try: + await asyncio.shield(query) + except asyncio.CancelledError: + continue + query.result() + return await execute(conn, sql, *args, **kwargs) + + async def identity(conn, name): + await original_identity(conn, name) + captured["conn"] = conn + known.set() + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(193789)") + raise ValueError("placement body failed before rollback") + + monkeypatch.setattr(cell_place, "_identity", identity) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + observer = await asyncpg.connect(stores["first"]) + task = asyncio.create_task(placement._local(stores, monkeypatch, "first")) + try: + await asyncio.wait_for(known.wait(), 3) + conn = captured["conn"] + pid = conn.get_server_pid() + await waiting(observer, pid) + started = time.monotonic() + if interrupt != "deadline": + task.cancel() + if interrupt == "second_cancel": + # A second signal arrives while the same operation is still active. + await asyncio.sleep(0) + task.cancel() + done, _ = await asyncio.wait({task}, timeout=0.8) + assert done, "placement must close its backend before waiting for rollback unwind" + with pytest.raises(TimeoutError if interrupt == "deadline" else asyncio.CancelledError): + await task + assert time.monotonic() - started < 0.8 + await assert_released(observer, pid) + finally: + # Exact owned test resources only; even a deliberately failing old owner + # cannot leave its query or scratch database alive after the regression. + if "conn" in captured and not captured["conn"].is_closed(): + with contextlib.suppress(Exception): + await captured["conn"].close(timeout=1) + if not task.done(): + task.cancel() + await asyncio.gather(task, *queries, return_exceptions=True) + await observer.close(timeout=1) + pg.expect_deployment(None) diff --git a/tests/integration/test_cell_place_durability.py b/tests/integration/test_cell_place_durability.py new file mode 100644 index 0000000000..6a1c1416a4 --- /dev/null +++ b/tests/integration/test_cell_place_durability.py @@ -0,0 +1,338 @@ +"""Placement retries survive promotion of a private physical standby. + +Only the dedicated CI drill enables this file. Its pair is separate from the +integration server's stores because it pauses replication and stops the primary. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import os +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlsplit + +import asyncpg +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg, provider_process +from server.core.schema import pg_migrate +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-place-drill-pg" +STANDBY = PRIMARY + "-standby" +ORG = "placement-durability" +DEPLOYMENT = "placement-drill" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +@dataclass +class Drill: + observer: Any + directory_dsn: str + local_dsn: str + tasks: list[asyncio.Task] = field(default_factory=list) + paused: bool = False + + def task(self, work: Any) -> asyncio.Task: + task = asyncio.create_task(work) + self.tasks.append(task) + return task + + async def pause(self) -> None: + await _docker("pause", STANDBY) + self.paused = True + + async def resume(self) -> None: + await _docker("unpause", STANDBY) + self.paused = False + + async def waiting(self, task: asyncio.Task, application: str) -> int: + async with asyncio.timeout(5): + while True: + assert not task.done(), "placement acknowledged before the standby resumed" + pid = await self.observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + application, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.02) + + async def seed(self, dsn: str, sql: str, *, marked: bool = False) -> None: + """Cancel a standby-blocked COMMIT and drain its acknowledgement before returning.""" + application = f"placement-seed-{len(self.tasks)}" + conn = await asyncpg.connect( + dsn, + timeout=5, + command_timeout=5, + server_settings={ + "application_name": application, + "synchronous_commit": "on", + **(pg.marker_settings() if marked else {}), + }, + ) + try: + await conn.execute("BEGIN") + await conn.execute(sql) + commit = self.task(conn.execute("COMMIT")) + await self.waiting(commit, application) + commit.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(commit, 5) + # Client cancellation precedes the server releasing the transaction. + # Drain it before termination can cancel the driver's CancelRequest. + await conn.close(timeout=5) + finally: + conn.terminate() + + +async def _cleanup_drill(state: Drill | None, observer: Any) -> None: + errors: list[BaseException] = [] + if state is not None: + try: + if state.paused: + await state.resume() + except BaseException as exc: # noqa: BLE001 — remaining owned cleanup must run + errors.append(exc) + for task in state.tasks: + if not task.done(): + task.cancel() + deadline = asyncio.get_running_loop().time() + 15 + for close in ( + lambda: asyncio.gather(*state.tasks, return_exceptions=True), + directory.close, + pg.close_pool, + ): + try: + async with asyncio.timeout_at(deadline): + await close() + except BaseException as exc: # noqa: BLE001 — attempt each close within the shared bound + errors.append(exc) + try: + observer.terminate() + except BaseException as exc: # noqa: BLE001 — preserve the first cleanup failure + errors.append(exc) + if errors: + raise errors[0] + + +@pytest.fixture +async def drill(monkeypatch): + if not os.environ.get("LOOM_CELL_PLACE_DRILL"): + pytest.skip("LOOM_CELL_PLACE_DRILL not set: requires its disposable physical standby") + assert os.environ["LOOM_CELL_PLACE_DRILL"] == "1" + dsn = os.environ["LOOM_CELL_PLACE_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + state = None + try: + # Refuse to disrupt a container unless the explicit DSN identifies this exact pair. + identifier = str( + await observer.fetchval("SELECT system_identifier FROM pg_control_system()") + ) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + _, directory_dsn = await create_database(dsn) + _, local_dsn = await create_database(dsn) + state = Drill(observer, directory_dsn, local_dsn) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(settings, "router_pg_dsn", directory_dsn) + monkeypatch.setattr(settings, "pg_dsn", local_dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_statement_timeout_ms", 10_000) + previous_expected = pg._expected + previous_orgs = cell._code_orgs + pg.expect_deployment(None) + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + try: + async with asyncio.timeout(60): + await _initialize(state) + yield state + finally: + pg.expect_deployment(previous_expected) + cell._code_orgs = previous_orgs + except BaseException: # Cleanup must not replace the setup failure. + with contextlib.suppress(BaseException): + await _cleanup_drill(state, observer) + raise + else: + await _cleanup_drill(state, observer) + + +async def _initialize(drill: Drill) -> None: + @contextlib.asynccontextmanager + async def connect(): + conn = await asyncpg.connect(drill.local_dsn, command_timeout=10) + try: + yield conn + finally: + conn.terminate() + + await pg_migrate.migrate(connect=connect) + await directory.migrate() + fingerprint = cell.as_value(cell.fingerprint(DEPLOYMENT)) + async with connect() as conn: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + DEPLOYMENT, + ) + await conn.execute("INSERT INTO loom.cell_first (is_first, cell) VALUES (true, 'first')") + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", json.dumps(fingerprint) + ) + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ('first', true, 'first.example.test', 'serving', 'enforced', $1, $2, $3)", + "3" * 64, + DEPLOYMENT, + fingerprint, + ) + + +async def _directory_retry(drill: Drill) -> None: + await drill.pause() + await drill.seed( + drill.directory_dsn, + "INSERT INTO loom_router.placements (org, cell) VALUES ('placement-durability', 'first'); " + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = 'first'", + ) + assert await directory.lookup(ORG) == ("first", "first.example.test") + retry = drill.task(directory.place(ORG, "first", lambda _: "", actor="operator")) + await drill.waiting(retry, "loom-router") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == ("first", DEPLOYMENT) + + +async def _local_retry(drill: Drill, monkeypatch) -> None: + async def verified_directory(router, name, org, deployment, fingerprint): + # The directory retry above already acknowledged this winner's physical flush. + assert (name, org, deployment) == ("first", ORG, DEPLOYMENT) + assert await directory.lookup(org) == (name, "first.example.test") + + monkeypatch.setattr(cell_place, "_directory", verified_directory) + await drill.pause() + await drill.seed( + drill.local_dsn, + "INSERT INTO loom.tenancy (org) VALUES ('placement-durability')", + marked=True, + ) + assert ( + await _sql( + PRIMARY, + urlsplit(drill.local_dsn).path.removeprefix("/"), + "SELECT org FROM loom.tenancy", + ) + == ORG + ) + args = argparse.Namespace(cell="first", org=ORG) + application = provider_process.participant_tag(provider_process.process_identity()) + cancelled = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + pid = await drill.waiting(cancelled, application) + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(cancelled, cell_place._bound_s() + 1) + async with asyncio.timeout(5): + while True: + if not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.02) + assert not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ) + retry = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + await drill.waiting(retry, application) + await drill.resume() + assert await asyncio.wait_for(retry, 5) == { + "org": ORG, + "cell": "first", + "deployment_id": DEPLOYMENT, + } + + +@pytest.mark.recovery +@pytest.mark.destructive +async def test_duplicate_placement_survives_physical_standby_promotion(drill, monkeypatch): + async with asyncio.timeout(120): + await _directory_retry(drill) + await _local_retry(drill, monkeypatch) + await directory.close() + await pg.close_pool() + drill.observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + directory_db = urlsplit(drill.directory_dsn).path.removeprefix("/") + local_db = urlsplit(drill.local_dsn).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT p.org, p.cell, c.deployment_id, c.placed FROM loom_router.placements p " + "JOIN loom_router.cells c USING (cell)", + ) + == f"{ORG}|first|{DEPLOYMENT}|1" + ) + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT actor, action, detail->>'org', detail->>'cell' FROM loom_router.audit", + ) + == f"operator|place|{ORG}|first" + ) + assert await _sql(STANDBY, local_db, "SELECT org FROM loom.tenancy") == ORG diff --git a/tests/integration/test_cell_update_addresses.py b/tests/integration/test_cell_update_addresses.py new file mode 100644 index 0000000000..c8d49de949 --- /dev/null +++ b/tests/integration/test_cell_update_addresses.py @@ -0,0 +1,172 @@ +"""Address refresh proves listeners before the generation-fenced directory write.""" + +from __future__ import annotations + +import hashlib +from dataclasses import replace + +import httpx +import pytest + +from server.router import app, directory +from server.settings import settings +from tests.integration.test_cell_update_directory import configured as configured +from tests.integration.test_cell_update_directory import observed +from tests.integration.test_router_directory_pg import dsn as dsn + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first", [False, True]) +async def test_address_refresh_changes_only_verified_addresses(configured, monkeypatch, first): + name = "first" if first else "other" + token = "lco_address-proof" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + monkeypatch.setattr(settings, "router_upstream_url", "http://new-main") + monkeypatch.setattr(settings, "router_upstream_probe_url", "http://new-probe") + expected = await observed(name) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + + def network(request): + if request.url.host == "new-main": + return httpx.Response(200, json={"deployment_id": f"dep-{name}"}) + return httpx.Response( + 200, + json={ + "deployment_id": f"dep-{name}", + "bucket": {"name": "", "prefix": ""}, + "nonces": nonces, + }, + ) + + before = await directory.attach_read(name) + router = app.Router(probe_transport=httpx.MockTransport(network)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app(router)), base_url="http://test" + ) as client: + response = await client.post( + f"/cells/v1/operator/cells/{name}/addresses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + "probe_url": "http://new-probe", + "upstream_url": "http://new-main" if first else before.upstream_url, + "nonces": nonces, + }, + ) + assert response.status_code == 200, response.text + after = await directory.attach_read(name) + assert after == replace( + before, + probe_url="http://new-probe", + upstream_url="http://new-main" if first else before.upstream_url, + ) + + +@pytest.mark.asyncio +async def test_stale_address_generation_cannot_overwrite_a_later_refresh(configured): + expected = await observed("first") + await observed("first") + before = await directory.attach_read("first") + with pytest.raises(directory.OperatorRefused, match="generation"): + await directory.update_addresses( + "first", "http://probe", "http://main", expected=expected, configured=lambda: True + ) + assert await directory.attach_read("first") == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "invalid", ["main-port", "deployment", "bucket", "nonce", "upstream", "config-changed"] +) +async def test_invalid_address_proof_leaves_exact_directory_unchanged( + configured, monkeypatch, invalid +): + token = "lco_address-refusal" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + monkeypatch.setattr(settings, "router_upstream_url", "http://new-main") + monkeypatch.setattr(settings, "router_upstream_probe_url", "http://new-probe") + expected = await observed("first") + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + before = await directory.attach_read("first") + + def network(request): + if request.url.host == "new-main": + return httpx.Response( + 200, json={"deployment_id": "wrong" if invalid == "upstream" else "dep-first"} + ) + answer = { + "deployment_id": "dep-first", + "bucket": {"name": "", "prefix": ""}, + "nonces": nonces, + } + if invalid == "main-port": + answer.pop("nonces") + elif invalid == "deployment": + answer["deployment_id"] = "other" + elif invalid == "bucket": + answer["bucket"] = {"name": "other", "prefix": ""} + elif invalid == "nonce": + answer["nonces"] = nonces | {"postgres": "old"} + elif invalid == "config-changed": + monkeypatch.setattr(settings, "router_upstream_url", "http://changed-main") + return httpx.Response(200, json=answer) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app(router)), base_url="http://test" + ) as client: + response = await client.post( + "/cells/v1/operator/cells/first/addresses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + "probe_url": "http://new-probe", + "upstream_url": "http://new-main", + "nonces": nonces, + }, + ) + assert response.status_code == 409, response.text + assert await directory.attach_read("first") == before + + +@pytest.mark.asyncio +async def test_old_router_config_can_reverse_addresses_with_fresh_expected_row( + configured, monkeypatch +): + # The all-router rollout prerequisite cannot be inferred from a successful CAS. + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + token = "lco_router-rollout" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + + def network(request): + answer = {"deployment_id": "dep-first"} + if "probe" in request.url.host: + answer |= {"bucket": {"name": "", "prefix": ""}, "nonces": nonces} + return httpx.Response(200, json=answer) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app(router)), base_url="http://test" + ) as client: + for version in ("new", "old"): + monkeypatch.setattr(settings, "router_upstream_url", f"http://{version}-main") + monkeypatch.setattr(settings, "router_upstream_probe_url", f"http://{version}-probe") + expected = await observed("first") + response = await client.post( + "/cells/v1/operator/cells/first/addresses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + "probe_url": f"http://{version}-probe", + "upstream_url": f"http://{version}-main", + "nonces": nonces, + }, + ) + assert response.status_code == 200, response.text + assert (await directory.attach_read("first")).upstream_url == "http://old-main" diff --git a/tests/integration/test_cell_update_directory.py b/tests/integration/test_cell_update_directory.py new file mode 100644 index 0000000000..cd247d5929 --- /dev/null +++ b/tests/integration/test_cell_update_directory.py @@ -0,0 +1,472 @@ +"""Configuration reservations, issuer routing and configured placements commit together.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from contextlib import asynccontextmanager +from typing import Any + +import httpx +import pytest + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.integration.test_router_directory_pg import dsn as dsn + + +def fingerprint(name: str, issuer: str, orgs: tuple[str, ...] = ()) -> dict[str, Any]: + return cell.as_value( + { + "deployment_id": f"dep-{name}", + "public_host": f"{name}.example.test", + "oauth_issuer_host": f"{name}.example.test", + "bearer_issuers": [issuer], + "configured_orgs": list(orgs), + } + ) + + +async def observed(name: str, *, claim: bool = True) -> directory.Control: + found = await directory.attach_read(name) + expected = directory.Control( + { + key: getattr(found.row, key) if hasattr(found.row, key) else getattr(found, key) + for key in directory.CONTROL_FIELDS + }, + found.row.mutation_generation, + ) + if claim: + found = await directory.claim(name, directory.Control(expected.values, None)) + expected = directory.Control(expected.values, found.row.mutation_generation) + return expected + + +@pytest.fixture +async def configured(dsn: str): + import asyncpg + + await directory.migrate() + conn = await asyncpg.connect(dsn) + try: + for name in ("first", "other"): + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, is_first, host, state, mode, credential_sha256, deployment_id, " + "current_fp, keys_fp, issuers, attach_started_at) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6::jsonb, $7, $8, now())", + name, + name == "first", + f"{name}.example.test", + ("a" if name == "first" else "b") * 64, + f"dep-{name}", + json.dumps(fingerprint(name, name)), + "c" * 64, + [name], + ) + yield conn + finally: + await conn.close() + + +async def change( + target: dict[str, Any], action: str = "config", *, expected: directory.Control | None = None +): + return await directory.update_configuration( + "first", + target, + action, + keys_fp="c" * 64, + refusal=app.placement_refusal, + expected=expected or await observed("first"), + ) + + +async def stored(conn: Any) -> dict[str, Any]: + row = await conn.fetchrow( + "SELECT current_fp, next_fp, issuers, placed, state, mode FROM loom_router.cells WHERE cell='first'" + ) + return dict(row) + + +@pytest.mark.asyncio +async def test_update_reserves_both_issuers_and_places_configured_orgs_once( + configured: Any, +) -> None: + target = fingerprint("first", "new", ("configured-a", "configured-b")) + await change(target) + row = await stored(configured) + assert json.loads(row["current_fp"]) == fingerprint("first", "first") + assert json.loads(row["next_fp"]) == target + assert row["issuers"] == ["first", "new"] + assert row["placed"] == 2 + assert (row["state"], row["mode"]) == ("serving", "enforced") + await configured.execute( + "UPDATE loom_router.cells SET closed=true, placements_max=2 WHERE cell='first'" + ) + await change(target) + assert await stored(configured) == row + assert ( + await configured.fetchval("SELECT count(*) FROM loom_router.audit WHERE action='place'") + == 2 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blocked", ["closed", "quota", "other-cell", "overlap", "root-keys"]) +async def test_update_refusal_leaves_no_partial_reservation_or_placement( + configured: Any, blocked: str +) -> None: + target = fingerprint("first", "new", ("a-new", "z-blocked")) + if blocked == "closed": + await configured.execute("UPDATE loom_router.cells SET closed=true WHERE cell='first'") + elif blocked == "quota": + await configured.execute("UPDATE loom_router.cells SET placements_max=1 WHERE cell='first'") + elif blocked == "other-cell": + await configured.execute( + "INSERT INTO loom_router.placements (org,cell) VALUES ('z-blocked','other')" + ) + elif blocked == "overlap": + target["fields"]["bearer_issuers"] = ["other"] + else: + await configured.execute( + "UPDATE loom_router.cells SET keys_fp=$1 WHERE cell='first'", "d" * 64 + ) + before = await stored(configured) + with pytest.raises((directory.OperatorRefused, directory.PlacementRefused)): + await change(target) + assert await stored(configured) == before + assert ( + await configured.fetchval("SELECT count(*) FROM loom_router.placements WHERE cell='first'") + == 0 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action,target_issuer", [("finish", "new"), ("revert", "first")]) +async def test_update_narrowing_retains_permanent_placements( + configured: Any, action: str, target_issuer: str +) -> None: + pending = fingerprint("first", "new", ("permanent",)) + await change(pending) + target = pending if action == "finish" else fingerprint("first", "first") + await change(target, action) + row = await stored(configured) + assert json.loads(row["current_fp"]) == target + assert row["next_fp"] is None + assert row["issuers"] == [target_issuer] + assert row["placed"] == 1 + assert ( + await configured.fetchval("SELECT cell FROM loom_router.placements WHERE org='permanent'") + == "first" + ) + await change(target, action) + assert await stored(configured) == row + + +@pytest.mark.asyncio +async def test_update_old_generation_cannot_narrow_after_another_claim(configured: Any) -> None: + target = fingerprint("first", "new") + await change(target) + stale = await observed("first") + await observed("first") + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="generation"): + await change(target, "finish", expected=stale) + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_concurrent_cells_cannot_reserve_one_issuer(configured: Any) -> None: + first, other = await observed("first"), await observed("other") + results = await asyncio.wait_for( + asyncio.gather( + change(fingerprint("first", "contested", ("first-org",)), expected=first), + directory.update_configuration( + "other", + fingerprint("other", "contested", ("other-org",)), + "config", + keys_fp="c" * 64, + refusal=app.placement_refusal, + expected=other, + ), + return_exceptions=True, + ), + timeout=15, + ) + assert sum(isinstance(result, directory.Attaching) for result in results) == 1 + assert sum(isinstance(result, directory.OperatorRefused) for result in results) == 1 + assert await configured.fetchval("SELECT sum(placed) FROM loom_router.cells") == 1 + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + + +@pytest.mark.asyncio +async def test_unreadable_other_reservation_cannot_be_replaced(configured: Any) -> None: + await configured.execute( + "UPDATE loom_router.cells SET next_fp=$1::jsonb WHERE cell='other'", '{"v":2,"fields":{}}' + ) + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="unreadable"): + await change(fingerprint("first", "new", ("unplaced",))) + assert await stored(configured) == before + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + + +@pytest.mark.asyncio +async def test_attaching_cell_keeps_issuers_unroutable(configured: Any) -> None: + await configured.execute( + "UPDATE loom_router.cells SET state='attaching', issuers='{}' WHERE cell='first'" + ) + await change(fingerprint("first", "new", ("configured",))) + row = await stored(configured) + assert row["issuers"] == [] + assert row["state"] == "attaching" + assert row["placed"] == 1 + + +@pytest.mark.asyncio +async def test_later_attaching_cell_refuses_configured_org_changes(configured: Any) -> None: + await configured.execute("UPDATE loom_router.cells SET state='attaching' WHERE cell='other'") + with pytest.raises(directory.OperatorRefused, match="later cell"): + await directory.update_configuration( + "other", + fingerprint("other", "new", ("unplaced",)), + "config", + keys_fp="c" * 64, + refusal=app.placement_refusal, + expected=await observed("other"), + ) + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + assert ( + await configured.fetchval("SELECT next_fp FROM loom_router.cells WHERE cell='other'") + is None + ) + + +@pytest.mark.asyncio +async def test_update_cannot_replace_another_pending_value(configured: Any) -> None: + await change(fingerprint("first", "new")) + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="pending"): + await change(fingerprint("first", "third", ("unplaced",))) + assert await stored(configured) == before + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + + +@pytest.mark.asyncio +async def test_recording_attach_leaves_original_configured_placement_to_attach( + configured: Any, +) -> None: + original = fingerprint("first", "first", ("served-before-attach",)) + await configured.execute( + "UPDATE loom_router.cells SET state='attaching', mode='recording', current_fp=$1::jsonb, issuers='{}' WHERE cell='first'", + json.dumps(original), + ) + await change(fingerprint("first", "new", ("served-before-attach",))) + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + assert (await stored(configured))["placed"] == 0 + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="pending"): + await change(fingerprint("first", "third", ("served-before-attach", "new-org"))) + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_update_does_not_rebuild_missing_serving_directory_placement(configured: Any) -> None: + original = fingerprint("first", "first", ("missing-placement",)) + await configured.execute( + "UPDATE loom_router.cells SET current_fp=$1::jsonb WHERE cell='first'", json.dumps(original) + ) + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="placement.*missing"): + await change(fingerprint("first", "new", ("missing-placement",))) + assert await stored(configured) == before + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + + +@pytest.mark.asyncio +async def test_configuration_fits_the_existing_operator_statement_budget( + configured: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected = await observed("first") + original = directory._connection + statements = [] + + @asynccontextmanager + async def counted(): + async with original() as conn: + conn.add_query_logger(lambda record: statements.append(record.query)) + yield conn + + monkeypatch.setattr(directory, "_connection", counted) + await change( + fingerprint("first", "new", tuple(f"configured-{i}" for i in range(40))), expected=expected + ) + await asyncio.sleep(0) # asyncpg schedules completed-query observers on the loop. + assert len(statements) == 9, statements + assert directory._bound_s(len(statements)) <= directory.CONTROL_MAX_S + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 40 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proof", ["ok", "refused", "new-claim"]) +async def test_configuration_http_requires_store_proof_before_any_change( + configured: Any, + monkeypatch: pytest.MonkeyPatch, + proof: str, +) -> None: + from starlette.responses import JSONResponse + + token = "lco_configuration-test" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + checked = [] + + async def stores_check(self, found, nonces): + checked.append((found.row.cell, nonces)) + if proof == "new-claim": + await observed("first") + return ( + JSONResponse({"error": "proof refused"}, status_code=409) + if proof == "refused" + else None + ) + + monkeypatch.setattr(app.Router, "_stores_check", stores_check) + expected = await observed("first") + target = fingerprint("first", "new", ("http-org",)) + nonces = {"postgres": "pg-nonce", "clickhouse": "ch-nonce", "bucket": None} + before = await stored(configured) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app()), base_url="http://test" + ) as client: + response = await client.post( + "/cells/v1/operator/cells/first/configuration", + headers={"Authorization": f"Bearer {token}"}, + json={ + "action": "config", + "target_fp": target, + "keys_fp": "c" * 64, + "nonces": nonces, + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + }, + ) + assert response.status_code == (200 if proof == "ok" else 409), response.text + assert checked == [("first", nonces)] + if proof == "ok": + assert response.json()["next_fp"] == target + assert ( + await configured.fetchval( + "SELECT cell FROM loom_router.placements WHERE org='http-org'" + ) + == "first" + ) + else: + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_configuration_accepts_the_actual_instance_key_fingerprint(configured, monkeypatch): + from server.core.deployment_config import fingerprints as instance_fingerprints + + keys = instance_fingerprints(settings)["keys_fp"] + assert len(keys) == 32 + await configured.execute("UPDATE loom_router.cells SET keys_fp=$1 WHERE cell='first'", keys) + token = "lco_configuration-real-keys" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + + async def stores_check(*_): + return None + + monkeypatch.setattr(app.Router, "_stores_check", stores_check) + compare = await observed("first") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app()), base_url="http://test" + ) as client: + response = await client.post( + "/cells/v1/operator/cells/first/configuration", + headers={"Authorization": f"Bearer {token}"}, + json={ + "action": "config", + "target_fp": fingerprint("first", "new"), + "keys_fp": keys, + "nonces": {"postgres": "pg", "clickhouse": "ch", "bucket": None}, + "expected": compare.values | {"mutation_generation": str(compare.generation)}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["next_fp"] == fingerprint("first", "new") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("winner", ["first", "other", None]) +async def test_configuration_placement_read_proves_actual_winners_without_mutation( + configured, winner +): + target = fingerprint("first", "new", ("configured",)) + await configured.execute( + "UPDATE loom_router.cells SET next_fp=$1::jsonb WHERE cell='first'", json.dumps(target) + ) + if winner is not None: + await configured.execute( + "INSERT INTO loom_router.placements(org,cell) VALUES ('configured',$1)", winner + ) + await configured.execute( + "UPDATE loom_router.cells SET placed=placed+1 WHERE cell=$1", winner + ) + before = await stored(configured) + compare = await observed("first") + if winner == "other": + with pytest.raises(directory.OperatorRefused, match="other"): + await directory.configuration_placements( + "first", ["configured"], expected=compare, keys_fp="c" * 64 + ) + else: + assert await directory.configuration_placements( + "first", ["configured"], expected=compare, keys_fp="c" * 64 + ) == (["configured"] if winner else []) + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_placement_proof_refuses_root_secret_drift_before_local_repair(configured): + with pytest.raises(directory.OperatorRefused, match="root-secret"): + await directory.configuration_placements( + "first", [], expected=await observed("first"), keys_fp="changed" + ) + + +@pytest.mark.asyncio +async def test_placement_proof_read_is_a_bounded_durable_noop(configured, monkeypatch): + expected = await observed("first") + original = directory._connection + statements = [] + before = await stored(configured) + wal_before = await configured.fetchval("SELECT pg_current_wal_insert_lsn()") + + @asynccontextmanager + async def counted(): + async with original() as conn: + conn.add_query_logger(lambda record: statements.append(record.query)) + yield conn + + monkeypatch.setattr(directory, "_connection", counted) + assert ( + await directory.configuration_placements("first", [], expected=expected, keys_fp="c" * 64) + == [] + ) + await asyncio.sleep(0) + assert len(statements) == 9, statements + assert directory._bound_s(len(statements)) == 120 + assert directory.CONTROL_MAX_S == 131 + assert directory._bound_s(len(statements)) <= directory.CONTROL_MAX_S + assert sum("pg_logical_emit_message" in sql for sql in statements) == 1 + assert statements[0] == "BEGIN;" and statements[-1] == "COMMIT;" + assert await configured.fetchval("SELECT pg_current_wal_insert_lsn()") > wal_before + assert await stored(configured) == before diff --git a/tests/integration/test_cell_update_participants_ch.py b/tests/integration/test_cell_update_participants_ch.py new file mode 100644 index 0000000000..48e81c9495 --- /dev/null +++ b/tests/integration/test_cell_update_participants_ch.py @@ -0,0 +1,153 @@ +"""Update reads a scratch participant census and waits for actual stored successors.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest + +from server.cli import cell_control +from server.cli import cell_update_participants as update +from server.core import cell, instances, provider_process +from server.core.ch import CHClient, query_tag +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_attach import FakeConn + + +@pytest.mark.asyncio +async def test_postcutoff_census_waits_for_certified_old_row_and_serving_successor(monkeypatch): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + from server.settings import settings + + database = "update_" + uuid4().hex + client = CHClient() + await client.connect() + with query_tag("test.cell_update.participants"): + async with contextlib.AsyncExitStack() as cleanup: + cleanup.push_async_callback(client.disconnect) + try: + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + cleanup.push_async_callback( + client.execute, "DROP DATABASE {db:Identifier} SYNC", {"db": database} + ) + await client.execute( + "CREATE TABLE {db:Identifier}._provider_participants AS loom._provider_participants", + {"db": database}, + ) + for table in ("instances", "cell_heartbeats"): + await client.execute( + f"CREATE TABLE {{db:Identifier}}.{table} AS loom.{table} ENGINE=MergeTree ORDER BY instance_id", + {"db": database}, + ) + monkeypatch.setattr(settings, "ch_database", database) + monkeypatch.setattr(settings, "ch_cluster", "") + old, new = uuid4(), uuid4() + proof = provider_process.certified(provider_process.host_proof()) + await client.insert( + "loom._provider_participants", + [ + { + "participant_id": old, + "state": "running", + "version": 1, + "host_proof": proof, + } + ], + ) + await client.execute( + "ALTER TABLE loom._provider_participants UPDATE heartbeat_at=now64(3)-toIntervalHour(25) WHERE 1 SETTINGS mutations_sync=2" + ) + initial = await update.census(client) + assert set(update.capture(list(initial.values()))) == {str(old)} + target = cell.as_value({"deployment_id": "target"}) + captured = asyncio.Event() + actual_census = update.census + reads = 0 + + async def census(ch): + nonlocal reads + result = await actual_census(ch) + reads += 1 + if reads == 1: + captured.set() + return result + + async def successor(): + await captured.wait() + await client.insert( + "loom._provider_participants", + [ + { + "participant_id": old, + "state": "retired", + "version": 2, + "host_proof": proof, + }, + { + "participant_id": new, + "state": "running", + "version": 1, + "host_proof": proof, + }, + ], + ) + instance = instances.row() | { + "instance_id": "successor", + "serving": 1, + "stopped": 0, + } + await client.insert("loom.instances", [instance]) + await client.insert( + "loom.cell_heartbeats", + [ + { + "instance_id": "successor", + "cell": "first", + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + target + | { + "participant_id": str(new), + "serves": True, + "marked": True, + "router": True, + } + ), + } + ], + ) + + monkeypatch.setattr(update, "census", census) + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + run = SimpleNamespace( + ch=client, + args=SimpleNamespace(cell="first", wait=5), + row={"is_first": True}, + owner=owner(FakeConn()), + ) + task = asyncio.create_task(successor()) + cleanup.push_async_callback(asyncio.gather, task, return_exceptions=True) + cleanup.callback(task.cancel) + await update.settle(run, target) + await task + assert reads >= 2 + assert len(await actual_census(client)) == 2 + # A workers-only heartbeat cannot replace the serving target. + await client.execute("TRUNCATE TABLE loom.instances") + await client.insert( + "loom.instances", [instances.row() | {"instance_id": "successor", "serving": 0}] + ) + assert not await update.replacement( + client, run, await actual_census(client), {str(old)}, target + ) + except BaseException: # Owned cleanup must not replace setup or body failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise diff --git a/tests/integration/test_cell_update_run_pg.py b/tests/integration/test_cell_update_run_pg.py new file mode 100644 index 0000000000..4ef957a07c --- /dev/null +++ b/tests/integration/test_cell_update_run_pg.py @@ -0,0 +1,481 @@ +"""The update coordinator commits real local and directory state on isolated stores.""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import json +from argparse import Namespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_control, cell_update_participants, cell_update_pg, cell_update_probe +from server.cli.cell_update_run import Run +from server.core import cell, deployment_config, pg +from server.router import app +from server.settings import settings +from tests.integration.test_cell_control_fence import control_stores as control_stores +from tests.integration.test_cell_control_fence import local_control as local_control +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_update_run import value + +REAL_POST = cell_update_probe.post +REAL_STORES_CHECK = app.Router._stores_check + + +async def _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before): + import asyncpg + + observer = await asyncpg.connect(local_dsn) + try: + await pg.init_connection(observer) + local = await cell_update_pg.read(observer, "first") + assert local == before + assert await observer.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", backend_pid + ) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", backend_pid + ) + finally: + await observer.close() + recorded = await directory_conn.fetchrow("SELECT current_fp,next_fp FROM loom_router.cells") + assert json.loads(recorded["current_fp"]) == value() + assert json.loads(recorded["next_fp"]) == value("new", orgs=("permanent",)) + assert ( + await directory_conn.fetchval( + "SELECT cell FROM loom_router.placements WHERE org=$1", "permanent" + ) + == "first" + ) + + +@pytest.fixture +async def update_stores(control_stores, local_control, monkeypatch, tmp_path): + client, directory_conn, _ = control_stores + local_conn, local_dsn = local_control + token = "lco_" + "f" * 128 + monkeypatch.setattr(settings, "router_operator_token", token) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + current = value() + await directory_conn.execute( + "UPDATE loom_router.cells SET state='serving',mode='enforced', current_fp=$1::jsonb, " + "keys_fp=$2, issuers=ARRAY['old']", + json.dumps(current), + deployment_config.fingerprints(settings)["keys_fp"], + ) + await local_conn.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await local_conn.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('enforced')") + await local_conn.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1::jsonb)", current) + await local_conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings: {}\n") + events = [] + real_write = cell_update_pg.write + + async def write(owned, expected, row, target, pending, orgs): + events.append("local-write") + # A successor is valid only in a fresh owner after the original backend died. + assert owned.conn is local_conn or local_conn.is_closed() + # The observed directory reservation is already durable before local widening. + recorded = await directory_conn.fetchrow("SELECT current_fp,next_fp FROM loom_router.cells") + assert (json.loads(recorded["current_fp"]) if recorded["current_fp"] else None) == row[ + "current_fp" + ] + assert (json.loads(recorded["next_fp"]) if recorded["next_fp"] else None) == row["next_fp"] + return await real_write(owned, expected, row, target, pending, orgs) + + async def post(run, route, body): + events.append("directory-" + body["action"]) + return await run.call( + route, + { + **body, + **cell_control.preconditions(run.row), + "nonces": {"postgres": "pg", "clickhouse": "ch", "bucket": None}, + }, + ) + + async def stores_checked(*_): + return None + + from server.router.app import Router + + monkeypatch.setattr(Router, "_stores_check", stores_checked) + monkeypatch.setattr(cell_update_probe, "post", post) + monkeypatch.setattr(cell_update_pg, "write", write) + + def make(action="config", target=None): + from server.cli import cell_update_config + + monkeypatch.setattr( + cell_update_config, + "target_fingerprint", + lambda *_: target or value("new", orgs=("permanent",)), + ) + args = Namespace( + cell="first", + config=path if action == "config" else None, + finish=action == "finish", + revert=action == "revert", + probe_url=None, + wait=0, + ) + return Run(None, args, httpx.URL("http://router"), client._transport) + + yield make, local_conn, directory_conn, owner(local_conn), events, local_dsn + + +@pytest.mark.asyncio +async def test_acceptance_error_rolls_back_tenancy_and_preserves_directory_retry(update_stores): + import asyncpg + + make, local_conn, directory_conn, owned, _, local_dsn = update_stores + backend_pid = local_conn.get_server_pid() + before = await cell_update_pg.read(local_conn, "first") + await local_conn.execute( + "CREATE FUNCTION reject_acceptance() RETURNS trigger LANGUAGE plpgsql AS $$ " + "BEGIN RAISE EXCEPTION 'acceptance refused by test'; END $$; " + "CREATE TRIGGER reject_acceptance BEFORE UPDATE ON loom.cell_self " + "FOR EACH ROW EXECUTE FUNCTION reject_acceptance()" + ) + with pytest.raises(asyncpg.RaiseError, match="acceptance refused by test"): + await make().start(owned) + assert owned.closed and local_conn.is_closed() + await _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before) + + +@pytest.mark.asyncio +async def test_cancelled_acceptance_releases_backend_and_rolls_back_tenancy(update_stores): + import asyncpg + + make, local_conn, directory_conn, owned, _, local_dsn = update_stores + backend_pid = local_conn.get_server_pid() + before = await cell_update_pg.read(local_conn, "first") + async with contextlib.AsyncExitStack() as cleanup: + try: + blocker = await asyncpg.connect(local_dsn) + cleanup.push_async_callback(blocker.close) + transaction = blocker.transaction() + await transaction.start() + cleanup.push_async_callback(transaction.rollback) + observer = await asyncpg.connect(local_dsn) + cleanup.push_async_callback(observer.close) + await blocker.fetchrow("SELECT id FROM loom.cell_self FOR UPDATE") + task = asyncio.create_task(make().start(owned)) + cleanup.push_async_callback(asyncio.gather, task, return_exceptions=True) + cleanup.callback(task.cancel) + async with asyncio.timeout(4): + while not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event_type='Lock' AND query LIKE 'UPDATE loom.cell_self%')", + backend_pid, + ): + assert not task.done() + await asyncio.sleep(0.005) + task.cancel("acceptance-stop") + with pytest.raises(asyncio.CancelledError, match="acceptance-stop"): + await asyncio.wait_for(task, timeout=4) + assert owned.closed and local_conn.is_closed() + except BaseException: # Every acquired resource must close before failure escapes. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise + await _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before) + + +@pytest.mark.asyncio +async def test_coordinator_config_and_retry_commit_both_real_stores_once(update_stores): + make, local_conn, directory_conn, owned, events, _ = update_stores + await make().start(owned) + await make().start(owned) + expected = value("new", orgs=("permanent",)) + local = await cell_update_pg.read(local_conn, "first") + assert local.current == value() and local.pending == expected + assert [dict(row) for row in await local_conn.fetch("SELECT org FROM loom.tenancy")] == [ + {"org": "permanent"} + ] + assert await directory_conn.fetchval("SELECT placed FROM loom_router.cells") == 1 + assert await directory_conn.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + assert events == ["directory-config", "local-write", "directory-config", "local-write"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["finish", "revert"]) +async def test_actual_cutoff_precedes_turnover_and_directory_release( + update_stores, monkeypatch, action +): + make, local_conn, directory_conn, owned, events, _ = update_stores + await make().start(owned) + # Lost local repair is reproduced by deleting only this scratch tenancy row. + await local_conn.execute("DELETE FROM loom.tenancy WHERE org='permanent'") + target = value("new", orgs=("permanent",)) if action == "finish" else value() + + async def barrier(run, wanted): + assert wanted == target + local = await cell_update_pg.read(local_conn, "first") + assert local.current == target and local.pending is None + assert await local_conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom.tenancy WHERE org='permanent')" + ) + assert await directory_conn.fetchval("SELECT next_fp IS NOT NULL FROM loom_router.cells") + events.append("turnover") + + monkeypatch.setattr(cell_update_participants, "settle", barrier) + await make(action).start(owned) + stored = await directory_conn.fetchrow( + "SELECT current_fp,next_fp,placed FROM loom_router.cells" + ) + assert json.loads(stored["current_fp"]) == target and stored["next_fp"] is None + assert stored["placed"] == 1 + assert events[-3:] == ["local-write", "turnover", "directory-" + action] + + +@pytest.mark.asyncio +async def test_directory_commit_then_local_failure_is_repaired_without_duplicate_placement( + update_stores, monkeypatch +): + make, local_conn, directory_conn, owned, _, _ = update_stores + real = cell_update_pg.write + monkeypatch.setattr( + cell_update_pg, "write", AsyncMock(side_effect=OSError("local unavailable")) + ) + with pytest.raises(OSError, match="local unavailable"): + await make().start(owned) + assert await directory_conn.fetchval("SELECT next_fp IS NOT NULL FROM loom_router.cells") + assert (await cell_update_pg.read(local_conn, "first")).pending is None + monkeypatch.setattr(cell_update_pg, "write", real) + await make().start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending == value( + "new", orgs=("permanent",) + ) + assert await directory_conn.fetchval("SELECT placed FROM loom_router.cells") == 1 + + +@pytest.mark.asyncio +async def test_turnover_timeout_leaves_real_directory_dual_and_local_single_for_retry( + update_stores, monkeypatch +): + make, local_conn, directory_conn, owned, _, _ = update_stores + await make().start(owned) + monkeypatch.setattr( + cell_update_participants, "settle", AsyncMock(side_effect=TimeoutError("old process")) + ) + with pytest.raises(TimeoutError, match="old process"): + await make("finish").start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending is None + assert await directory_conn.fetchval("SELECT next_fp IS NOT NULL FROM loom_router.cells") + monkeypatch.setattr(cell_update_participants, "settle", AsyncMock()) + await make("finish").start(owned) + assert await directory_conn.fetchval("SELECT next_fp IS NULL FROM loom_router.cells") + + +@pytest.mark.asyncio +async def test_prebegin_real_local_pair_only_never_reserves_or_places(update_stores): + make, local_conn, directory_conn, owned, events, _ = update_stores + await directory_conn.execute( + "UPDATE loom_router.cells SET state='attaching',current_fp=NULL,next_fp=NULL,issuers='{}'" + ) + await make().start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending == value( + "new", orgs=("permanent",) + ) + assert await directory_conn.fetchval( + "SELECT current_fp IS NULL AND next_fp IS NULL FROM loom_router.cells" + ) + assert await directory_conn.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + assert await local_conn.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert events == ["local-write"] + + +@pytest.mark.asyncio +async def test_changed_root_keys_refuse_before_real_local_cutoff(update_stores, monkeypatch): + make, local_conn, directory_conn, owned, events, _ = update_stores + await make().start(owned) + before = await cell_update_pg.read(local_conn, "first") + await directory_conn.execute("UPDATE loom_router.cells SET keys_fp='changed'") + barrier = AsyncMock(side_effect=AssertionError("must not reach barrier")) + monkeypatch.setattr(cell_update_participants, "settle", barrier) + with pytest.raises(cell_control.Refused, match="root-secret fingerprint changed"): + await make("finish").start(owned) + assert await cell_update_pg.read(local_conn, "first") == before + assert events.count("local-write") == 1 + barrier.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("aba", [False, True]) +async def test_delayed_actual_finish_cannot_release_after_backend_death_and_fresh_rewiden( + update_stores, monkeypatch, aba +): + import asyncio + + import asyncpg + + from server.router import directory + + make, local_conn, directory_conn, owned, _, dsn = update_stores + await make().start(owned) + monkeypatch.setattr(cell_update_participants, "settle", AsyncMock()) + entered, resume = asyncio.Event(), asyncio.Event() + real_change = directory.update_configuration + delayed_once = False + + async def delayed(*args, **kwargs): + nonlocal delayed_once + if args[2] == "finish" and not delayed_once: + delayed_once = True + entered.set() + await resume.wait() + return await real_change(*args, **kwargs) + + monkeypatch.setattr(directory, "update_configuration", delayed) + async with contextlib.AsyncExitStack() as cleanup: + try: + pending = asyncio.create_task(make("finish").start(owned)) + cleanup.push_async_callback(asyncio.gather, pending, return_exceptions=True) + cleanup.callback(pending.cancel) + cleanup.callback(resume.set) + admin = await asyncpg.connect(dsn) + cleanup.push_async_callback(admin.close) + successor = await asyncpg.connect(dsn) + cleanup.push_async_callback(successor.close) + await directory._init(successor) + await asyncio.wait_for(entered.wait(), 5) + old_generation = str( + await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") + ) + await admin.execute("SELECT pg_terminate_backend($1)", local_conn.get_server_pid()) + assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + fresh_owner = owner(successor) + await make().start(fresh_owner) + if aba: + await make("revert").start(fresh_owner) + await make().start(fresh_owner) + assert old_generation != str( + await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") + ) + assert (await cell_update_pg.read(successor, "first")).pending is not None + resume.set() + with pytest.raises(cell_control.Refused, match="generation|attach lock|lock session"): + await pending + stored = await directory_conn.fetchrow( + "SELECT current_fp,next_fp,placed FROM loom_router.cells" + ) + assert json.loads(stored["current_fp"]) == value() + assert json.loads(stored["next_fp"]) == value("new", orgs=("permanent",)) + assert stored["placed"] == 1 + assert (await cell_update_pg.read(successor, "first")).pending is not None + except BaseException: # Owned cleanup must preserve the original test failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise + + +@pytest.mark.asyncio +async def test_local_cas_refuses_changed_state_before_tenancy_repair(update_stores): + make, local_conn, _, owned, _, _ = update_stores + await make().start(owned) + initial = await cell_update_pg.read(local_conn, "first") + run = make("finish") + run.row = await run.call("") + await local_conn.execute("UPDATE loom.cell_self SET next_fp=NULL") + with pytest.raises(cell_control.Refused, match="acceptance changed"): + await cell_update_pg.write(owned, initial, run.row, initial.current, None, ["not-held"]) + # The owner discards the failed transaction; the command cannot reuse the backend. + assert owned.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proof", ["valid", "wrong", "busy"]) +async def test_update_uses_fresh_real_pg_ch_nonces_and_cleans_only_its_values( + update_stores, monkeypatch, proof +): + import os + from uuid import uuid4 + + from server.cli import cell_prepare + from server.core import cell_probe, leases + from server.core.ch import CHClient, query_tag + + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + make, local_conn, directory_conn, owned, _, _ = update_stores + database = "update_nonce_" + uuid4().hex + table = database + ".locks" + client = CHClient() + await client.connect() + with query_tag("test.cell_update.nonces"): + async with contextlib.AsyncExitStack() as cleanup: + cleanup.push_async_callback(client.disconnect) + try: + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + cleanup.push_async_callback( + client.execute, "DROP DATABASE {db:Identifier} SYNC", {"db": database} + ) + await client.execute( + "CREATE TABLE {db:Identifier}.locks (name String, holder String, " + "acquired_at DateTime64(3), renewed_at DateTime64(3), ttl_s UInt32, " + "build_commit String) ENGINE=KeeperMap({path:String}) PRIMARY KEY name", + {"db": database, "path": "/loom/test/" + database}, + ) + monkeypatch.setattr( + leases, "LOCK_RELEASE", leases.LOCK_RELEASE.replace(leases.LOCK_TABLE, table) + ) + monkeypatch.setattr(leases, "LOCK_TABLE", table) + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(cell_update_probe, "post", REAL_POST) + monkeypatch.setattr(app.Router, "_stores_check", REAL_STORES_CHECK) + monkeypatch.setattr(cell_prepare, "_sleep", AsyncMock()) + await directory_conn.execute( + "UPDATE loom_router.cells SET probe_url='http://probe',upstream_url='http://main'" + ) + requests = [] + + async def network(request): + if request.url.host == "main": + return httpx.Response(200, json={"deployment_id": "dep-first"}) + requests.append(request) + actual = await cell_probe.nonces() + assert actual["postgres"] and actual["clickhouse"] + if proof == "busy" and len(requests) == 1: + return httpx.Response(429, headers={"Retry-After": "1"}) + return httpx.Response( + 200, + json={ + "deployment_id": "dep-first", + "bucket": {"name": "", "prefix": ""}, + "nonces": actual | ({"postgres": "old"} if proof == "wrong" else {}), + }, + ) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + run = make() + run.transport = httpx.ASGITransport(app=app.build_app(router)) + run.ch, run.args.wait = client, 3 + if proof == "wrong": + with pytest.raises(cell_prepare.Refused, match="does not read the stores"): + await run.start(owned) + assert await directory_conn.fetchval( + "SELECT next_fp IS NULL FROM loom_router.cells" + ) + assert (await cell_update_pg.read(local_conn, "first")).pending is None + else: + await run.start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending is not None + assert len(requests) == (2 if proof == "busy" else 1) + assert await cell_probe.nonces() == { + "postgres": None, + "clickhouse": None, + "bucket": None, + } + except BaseException: # Owned cleanup must not replace setup or body failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise diff --git a/tests/integration/test_cell_update_scim_live_ch.py b/tests/integration/test_cell_update_scim_live_ch.py new file mode 100644 index 0000000000..ea82530bba --- /dev/null +++ b/tests/integration/test_cell_update_scim_live_ch.py @@ -0,0 +1,116 @@ +"""Update checks provisioning against the prepared cell's actual private stores.""" + +from __future__ import annotations + +import sys +from argparse import Namespace + +import httpx +import pytest + +from server.auth import scim +from server.cli import cell_control, cell_update_pg +from server.cli.cell_update_run import Run +from server.core import cell, pg +from server.router import directory +from server.settings import settings +from tests.integration.test_cell_attach import ROUTER, _Network, _prepare +from tests.integration.test_cell_scim_census_live_ch import ( + cell_clickhouse_databases as cell_clickhouse_databases, +) +from tests.integration.test_cell_scim_census_live_ch import cell_stores as cell_stores +from tests.integration.test_cell_scim_census_live_ch import ( + genuine_cell_processes as genuine_cell_processes, +) +from tests.integration.test_cell_scim_census_live_ch import people as people +from tests.integration.test_scim_person_live_ch import People +from tests.unit.cell_control_owner import owner + +pytestmark = pytest.mark.usefixtures("ch_target_required") + + +@pytest.fixture(autouse=True) +async def _clean_people(people: People): + try: + yield people + finally: + first_error = sys.exception() + for sql, params in ( + ( + "ALTER TABLE loom.namespaces DELETE WHERE org = {org:String} " + "SETTINGS mutations_sync = 2", + {"org": people.org}, + ), + ( + "ALTER TABLE loom.scim_users DELETE WHERE id IN {ids:Array(String)} " + "SETTINGS mutations_sync = 2", + {"ids": [scim.user_id(email) for email in people.emails]}, + ), + ): + try: + await people.client.execute(sql, params=params) + except BaseException as error: # noqa: BLE001 — attempt every owned cleanup. + if first_error is None: + first_error = error + else: + first_error.add_note(f"identity cleanup also failed: {type(error).__name__}") + if first_error is not None: + raise first_error + + +@pytest.mark.parametrize("case", ["provisioned", "missing-person", "changed-local-pair"]) +async def test_update_scim_precondition_uses_actual_acceptance_and_people( + cell_stores, people: People, monkeypatch: pytest.MonkeyPatch, case: str +) -> None: + monkeypatch.setattr(settings, "scim_token", "") + monkeypatch.setattr(settings, "oauth_login_provider", "") + monkeypatch.setattr(settings, "auth_required", False) + async with _Network() as network: + await _prepare(cell_stores, network) + email = people.email("update-owner") + await people.account(email, email) + if case != "missing-person": + await people.provision(email) + monkeypatch.setattr(settings, "scim_token", "lsc_first_" + "a" * 64) + monkeypatch.setattr(settings, "oauth_login_provider", "oidc") + monkeypatch.setattr(settings, "auth_required", True) + args = Namespace(cell="first", revoke_unprovisioned=False) + run = Run(people.client, args, httpx.URL(ROUTER), network) + async with pg.dedicated() as conn: + await conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + run.owner = owner(conn) + run.row = await run.call("") + run.state = await cell_update_pg.read(conn, "first") + assert run.state.current["fields"]["scim"] is False + target = cell.as_value({**run.state.current["fields"], "scim": True}) + if case == "changed-local-pair": + await conn.execute("UPDATE loom.cell_self SET next_fp=$1::jsonb", target) + pool = await directory.pool() + async with pool.acquire() as directory_conn: + before = ( + await cell_update_pg.read(conn, "first"), + await conn.fetchrow("SELECT * FROM loom.cell_self"), + await conn.fetch("SELECT * FROM loom.tenancy ORDER BY org"), + await directory_conn.fetch("SELECT * FROM loom_router.cells ORDER BY cell"), + await directory_conn.fetch("SELECT * FROM loom_router.placements ORDER BY org"), + await directory_conn.fetch("SELECT * FROM loom_router.audit ORDER BY id"), + ) + if case == "provisioned": + await run.scim_precondition(target) + else: + reason = ( + "unprovisioned accounts subject" + if case == "missing-person" + else "accepted local fingerprint pair changed" + ) + with pytest.raises(cell_control.Refused, match=reason): + await run.scim_precondition(target) + after = ( + await cell_update_pg.read(conn, "first"), + await conn.fetchrow("SELECT * FROM loom.cell_self"), + await conn.fetch("SELECT * FROM loom.tenancy ORDER BY org"), + await directory_conn.fetch("SELECT * FROM loom_router.cells ORDER BY cell"), + await directory_conn.fetch("SELECT * FROM loom_router.placements ORDER BY org"), + await directory_conn.fetch("SELECT * FROM loom_router.audit ORDER BY id"), + ) + assert after == before diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py index 928517ddf3..552c16c8fe 100644 --- a/tests/integration/test_directory_control_cleanup.py +++ b/tests/integration/test_directory_control_cleanup.py @@ -345,9 +345,9 @@ async def codec(self, *args, **kwargs): raise original async def close(self, **kwargs): - if self is failed.get("raw"): - # This is the failed, never-borrowed raw codec session, not an active - # command session: synchronous quarantine must precede driver cleanup. + if self is failed.get("raw") and self.is_closed(): + # The initializer's first public close runs normally. Driver cleanup + # reaches this branch only after that session has been closed. assert self.is_closed() failed["driver close"] = True if driver_close == "exit": @@ -427,8 +427,8 @@ async def acquire(): result = await directory._control_result(acquire) assert isinstance(result, directory._ControlFatal) and result.error is original assert original.__notes__ == ["directory initialization session closure is unknown"] - # Termination's request is not the proof; here the driver's subsequent - # ordinary-error public close really completes and the observer proves it. + # The initializer's public close completes before termination fails; + # the observer still proves backend and lock absence. await released(directory_database, failed["pid"]) finally: monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index 5901f18837..e529b046dc 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -132,7 +132,8 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, org ) racing = asyncio.gather( - directory.place(org, a, _never), directory.place(org, b, _never) + directory.place(org, a, _never, actor="operator"), + directory.place(org, b, _never, actor="operator"), ) # Both placements wait on the org's lock before it is released. for _ in range(100): @@ -156,10 +157,11 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT actor FROM loom_router.audit WHERE action = 'place' AND detail->>'org' = $1", org, ) - assert [r["actor"] for r in audit] == [f"cell:{winners[0][0]}"] + assert [r["actor"] for r in audit] == ["operator", "operator"] # Ten concurrent placements in a cell with room for three: three are written. results = await asyncio.gather( - *(directory.place(f"{q}-{i}", q, _never) for i in range(10)), return_exceptions=True + *(directory.place(f"{q}-{i}", q, _never, actor="operator") for i in range(10)), + return_exceptions=True, ) assert sum(isinstance(r, tuple) for r in results) == 3 assert sum(isinstance(r, directory.PlacementRefused) for r in results) == 7 @@ -168,10 +170,13 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: assert held == 3 # An org placed already answers its cell, whatever the quota or the cell's state. placed = next(f"{q}-{i}" for i, r in enumerate(results) if isinstance(r, tuple)) - assert await directory.place(placed, q, lambda _r: "closed") == (q, f"dep-{q}") + assert await directory.place(placed, q, lambda _r: "closed", actor="operator") == ( + q, + f"dep-{q}", + ) # A refusal judged on the locked row writes nothing. with pytest.raises(directory.PlacementRefused, match="closed"): - await directory.place(f"new-{a}", a, lambda _r: "closed") + await directory.place(f"new-{a}", a, lambda _r: "closed", actor="operator") assert await directory.lookup(f"new-{a}") is None # Routing reads every cell with its upstream address ('' until attach writes it). cells, _aliases = await directory.routing() @@ -801,7 +806,9 @@ async def _held(*a: Any) -> None: conn = await asyncpg.connect(own_dsn) try: async with asyncio.timeout(30): - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await inserted.wait() batch = asyncio.ensure_future( directory.attach_place(name, dep, ["acme", "beta"], expected=await _observed(name)) @@ -851,12 +858,17 @@ async def hold(conn, found): directory.attach_place(name, dep, ["acme"], expected=await _observed(name)) ) await written.wait() - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await batch == (1, 1) assert await live == (name, dep) - assert await _placements_of(conn, name) == (1, [("operator", "acme")]) + assert await _placements_of(conn, name) == ( + 1, + [("operator", "acme"), (f"cell:{name}", "acme")], + ) finally: release.set() await conn.close() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index c250b4e04c..8db6694739 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {scim-token,credential,limit,which,remove,prepare,attach} ...", - "positional arguments: {scim-token,credential,limit,which,remove,prepare,attach} scim-token Generate a cell's SCIM token offline. credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", + "usage: loom cell [-h] {scim-token,credential,limit,which,remove,place,prepare,attach,update} ...", + "positional arguments: {scim-token,credential,limit,which,remove,place,prepare,attach,update} scim-token Generate a cell's SCIM token offline. credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve. update Reserve configuration, finish or revert its rollout, or refresh verified addresses.", "options: -h, --help show this help message and exit" ], "loom cell scim-token": [ @@ -464,6 +464,10 @@ "usage: loom cell remove [-h] --cell CELL [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL The cell to remove (its LOOM_CELL) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell place": [ + "usage: loom cell place [-h] --cell CELL --org ORG [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --org ORG The org to place --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom cell prepare": [ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" @@ -472,6 +476,11 @@ "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--revoke-unprovisioned] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; keep cell state, tenancy triggers and placements unchanged. Command participation is still recorded (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --revoke-unprovisioned Cut off pre-attach tokens of unprovisioned subjects; dry-run never revokes --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell update": [ + "usage: loom cell update [-h] --cell CELL [--config CONFIG | --finish | --revert | --probe-url PROBE_URL] [--revoke-unprovisioned] [--router ROUTER] [--wait WAIT]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --config CONFIG Nonsecret version: 1 settings YAML overlaid on this CLI's effective settings; does not install the file --finish Promote next after durable local narrowing and permanent participant turnover --revert Retain current after durable local narrowing and permanent participant turnover; keep placements --probe-url PROBE_URL Later cell's verified probe listener URL; no flag refreshes the first cell's router-configured addresses --revoke-unprovisioned Cut off pre-update tokens of subjects that cannot satisfy the target SCIM policy --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL) --wait WAIT Seconds to wait for probe readiness or participant turnover (0 to 86400, default: 600)", + "Finish/revert require foundation-capable runtimes and completed positive legacy-process turnover. A census cannot verify that external inventory. Address refresh requires every router updated first; retain old endpoints until every router refreshes or restarts after the directory change." + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index be494d55be..03d434256a 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12951,6 +12951,99 @@ } ] }, + "place": { + "prog": "loom cell place", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_place:cmd_place" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "org", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--org" + ], + "dest": "org", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The org to place", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] + }, "prepare": { "prog": "loom cell prepare", "usage": null, @@ -13312,6 +13405,189 @@ "metavar": null } ] + }, + "update": { + "prog": "loom cell update", + "usage": null, + "description": null, + "epilog": "Finish/revert require foundation-capable runtimes and completed positive legacy-process turnover. A census cannot verify that external inventory. Address refresh requires every router updated first; retain old endpoints until every router refreshes or restarts after the directory change.", + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_update:cmd_update" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "config", + "finish", + "revert", + "probe_url", + "revoke_unprovisioned", + "router", + "wait" + ] + } + ], + "mutually_exclusive": [ + { + "required": false, + "dests": [ + "config", + "finish", + "revert", + "probe_url" + ] + } + ], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--config" + ], + "dest": "config", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": false, + "help": "Nonsecret version: 1 settings YAML overlaid on this CLI's effective settings; does not install the file", + "metavar": null + }, + { + "option_strings": [ + "--finish" + ], + "dest": "finish", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Promote next after durable local narrowing and permanent participant turnover", + "metavar": null + }, + { + "option_strings": [ + "--revert" + ], + "dest": "revert", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Retain current after durable local narrowing and permanent participant turnover; keep placements", + "metavar": null + }, + { + "option_strings": [ + "--probe-url" + ], + "dest": "probe_url", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "Later cell's verified probe listener URL; no flag refreshes the first cell's router-configured addresses", + "metavar": null + }, + { + "option_strings": [ + "--revoke-unprovisioned" + ], + "dest": "revoke_unprovisioned", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Cut off pre-update tokens of subjects that cannot satisfy the target SCIM policy", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds to wait for probe readiness or participant turnover (0 to 86400, default: 600)", + "metavar": null + } + ] } }, "aliases_of": { @@ -13320,8 +13596,10 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", + "place": "loom cell place", "prepare": "loom cell prepare", - "attach": "loom cell attach" + "attach": "loom cell attach", + "update": "loom cell update" }, "choices_help": [ { @@ -13349,6 +13627,11 @@ "help": "Remove the directory row of a cell attaching with no placement.", "metavar": "remove" }, + { + "dest": "place", + "help": "Place an org in this cell and record its tenancy.", + "metavar": "place" + }, { "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", @@ -13358,6 +13641,11 @@ "dest": "attach", "help": "Attach the prepared first cell: place its orgs, serve.", "metavar": "attach" + }, + { + "dest": "update", + "help": "Reserve configuration, finish or revert its rollout, or refresh verified addresses.", + "metavar": "update" } ] } diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index 71ba8309ee..feb7b9f72e 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -765,7 +765,7 @@ def test_the_configured_orgs_hold_the_org_of_each_repository_entry_the_code_conf monkeypatch.setattr(settings, "code_repo", code_repo) monkeypatch.setattr(settings, "code_repos_json", repos_json) monkeypatch.setattr(cell, "_code_orgs", repos.operator_orgs) - kept = {r.org for r in repos._configured_from_env()} + kept = {r.org for r in repos._configured_from_env(settings, {})} assert set(cell.fingerprint("dep-1")["configured_orgs"]) == ({code_org} | kept) - {""} diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 5e4fc8733f..3ae4ab4fb4 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -19,6 +19,7 @@ ROOT = Path(__file__).resolve().parents[2] _CONNECTS = { "run_connected", + "run_control_connected", "connect_checked", "connect_declared", "check_acceptance", @@ -536,18 +537,36 @@ class _Client: def test_no_cli_command_connects_outside_the_one_module() -> None: stray = [] + native = [] + native_owners = { + ("server/cli/cell_control.py", "Owner._open", "self.connect"), + ("server/cli/cell_place.py", "place.opening", "owner.connect"), + } for path in sorted((ROOT / "server" / "cli").glob("*.py")): if path.name == "_ch.py": continue - for call in _calls(ast.parse(path.read_text())): + tree = ast.parse(path.read_text()) + parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)} + for call in _calls(tree): f = call.func if not (isinstance(f, ast.Attribute) and f.attr == "connect"): continue receiver = f.value if isinstance(receiver, ast.Name) and receiver.id in _OTHER_CLIENTS: continue + scope = [] + ancestor = parents.get(call) + while ancestor is not None: + if isinstance(ancestor, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + scope.append(ancestor.name) + ancestor = parents.get(ancestor) + callsite = (str(path.relative_to(ROOT)), ".".join(reversed(scope)), ast.unparse(f)) + if callsite in native_owners: + native.append(callsite) + continue stray.append(f"{path.relative_to(ROOT)}:{call.lineno}") assert not stray, f"connect through server/cli/_ch.py instead: {stray}" + assert sorted(native) == sorted(native_owners) def test_a_code_command_that_addresses_an_org_connects_for_it() -> None: diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 1921ad91d6..712c3bc205 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -252,7 +252,13 @@ async def body(owned): @pytest.mark.asyncio async def test_late_connection_is_discarded_without_starting_command_work(monkeypatch): + import asyncpg + from server.core import pg + from server.settings import settings + + monkeypatch.setattr(settings, "pg_dsn", "postgres://not-connected") + monkeypatch.setattr(pg, "_expected", None) connected, discarded = asyncio.Event(), asyncio.Event() @@ -263,7 +269,7 @@ def terminate(self): conn = Late() - async def connect(**_): + async def connect(*args, **kwargs): try: await asyncio.Future() except asyncio.CancelledError: @@ -274,7 +280,7 @@ async def connect(**_): pass return conn - monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(asyncpg, "connect", connect) monkeypatch.setattr(cell_control, "OPEN_S", 0.04) body = AsyncMock() with pytest.raises(TimeoutError): @@ -304,17 +310,23 @@ async def opening(self, _what): @pytest.mark.asyncio async def test_late_connection_arriving_inside_reserve_is_closed_once(monkeypatch): + import asyncpg + from server.core import pg + from server.settings import settings + + monkeypatch.setattr(settings, "pg_dsn", "postgres://not-connected") + monkeypatch.setattr(pg, "_expected", None) conn = FakeConn() - async def connect(**_): + async def connect(*args, **kwargs): try: await asyncio.Future() except asyncio.CancelledError: return conn - monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(asyncpg, "connect", connect) monkeypatch.setattr(cell_control, "OPEN_S", 0.08) work = AsyncMock() with pytest.raises(TimeoutError): diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index b7099740cd..b80384b90e 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -102,6 +102,7 @@ def test_cell_control_migrations_are_classified(): # Entries name every sink, including nested callbacks and dynamic table expressions. # A new sink, including another statement in an existing owner, requires classification. GUARDED_WRITERS = { + "server/cli/cell_update_pg.py": {("write.change", "UPDATE", "loom.cell_self"): 1}, "server/cli/cell_attach_pg.py": {("_mode_transaction", "UPDATE", "loom.tenancy_mode"): 1}, "server/cli/cell_control.py": { ("backout.clear_state", "ALTER TABLE", "loom.cell_first"): 3, @@ -114,9 +115,13 @@ def test_cell_control_migrations_are_classified(): ("_write_rows.write", "INSERT INTO", "loom.cell_self"): 1, }, "server/router/directory.py": { + ("update_addresses.change", "UPDATE", "loom_router.cells"): 1, ("set_limit.change", "UPDATE", "loom_router.cells"): 1, ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, + ("_configuration_commit", "INSERT INTO", "loom_router.placements"): 1, + ("_configuration_commit", "INSERT INTO", "loom_router.audit"): 2, + ("_configuration_commit", "UPDATE", "loom_router.cells"): 1, ("attach_begin.change_locked", "UPDATE", "loom_router.cells"): 1, ("attach_place.change_locked", "INSERT INTO", "loom_router.placements"): 1, ("attach_place.change_locked", "UPDATE", "loom_router.cells"): 1, @@ -143,7 +148,7 @@ def test_cell_control_migrations_are_classified(): ("register", "INSERT INTO", "loom_router.audit"): 1, ("_place_one", "INSERT INTO", "loom_router.placements"): 1, ("_place_one", "UPDATE", "loom_router.cells"): 1, - ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("place", "INSERT INTO", "loom_router.audit"): 1, ("_audit", "INSERT INTO", "loom_router.audit"): 1, }, } @@ -315,6 +320,39 @@ def function(path: str, name: str) -> ast.AST: @pytest.mark.parametrize( ("path", "name", "required"), [ + ( + "server/cli/cell_update_run.py", + "update", + ['cell_control.run("update", run.start)'], + ), + ( + "server/cli/cell_update_run.py", + "Run.start", + [ + "cell_control.claim(owner.conn, self.call, self.row)", + "local.paired(self.state, self.row)", + ], + ), + ( + "server/cli/cell_update_pg.py", + "write", + ["owner.transaction(change)"], + ), + ( + "server/cli/cell_update_pg.py", + "write.change", + ["cell_control.paired(conn, registered)", "read(conn, expected.cell)"], + ), + ( + "server/router/directory.py", + "update_addresses", + ["_attach_change(cell_name, expected, change)"], + ), + ( + "server/router/directory.py", + "configuration_placements", + ["_attach_change(cell_name, expected, read)"], + ), ( "server/cli/cell_prepare.py", "prepare", @@ -444,6 +482,15 @@ def function(path: str, name: str) -> ast.AST: "claim", ["_attach_change(cell_name, expected, change_locked, claim=True)"], ), + ( + "server/router/directory.py", + "update_configuration", + [ + "_control_transaction(locked, seconds=CONTROL_MAX_S)", + "control_refusal(found, expected)", + "_configuration_commit(conn, found, current, pending, new, action)", + ], + ), *[ ( "server/router/directory.py", @@ -501,9 +548,9 @@ def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): # Operator reads may audit, but cannot bypass control guards for a state write. assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) - assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( - found.functions["place"] - ) + assert "_place_one(conn, org, cell_name, candidate, refusal)" in calls(found.functions["place"]) + assert "_cell(locked)" in calls(found.functions["place"]) + assert "verify(candidate)" in calls(found.functions["place"]) assert "refusal(candidate)" in calls(found.functions["_place_one"]) for name in ("set_limit", "rotate_credential"): assert not any(call.startswith("claim(") for call in calls(found.functions[name])) diff --git a/tests/unit/test_cell_control_connection_owner.py b/tests/unit/test_cell_control_connection_owner.py new file mode 100644 index 0000000000..d2c3a69fd7 --- /dev/null +++ b/tests/unit/test_cell_control_connection_owner.py @@ -0,0 +1,225 @@ +"""Control owns each raw session before deployment and codec I/O can wait.""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import weakref +from unittest.mock import AsyncMock +from uuid import UUID + +import asyncpg +import httpx +import pytest + +from server.cli import cell_control, cell_place +from server.core import pg, provider_process, retirement +from server.settings import settings + + +class Connection: + def __init__(self): + self.owner = None + self.io = [] + self.events = [] + self.block_stage = None + self.error_stage = None + self.failure = ValueError("initialization refused") + self.started = asyncio.Event() + self.release = asyncio.Event() + self.close_started = asyncio.Event() + self.close_allowed = asyncio.Event() + self.close_allowed.set() + self.closes = 0 + self.terminations = 0 + self.closed = False + + async def step(self, stage): + cleanup_owner = getattr(self.owner, "budget", self.owner) + self.io.append((stage, self.owner.conn is self, cleanup_owner.conn is self)) + if stage == self.error_stage: + raise self.failure + if stage == self.block_stage: + self.started.set() + try: + await self.release.wait() + except asyncio.CancelledError: + self.events.append("initialization cancelled") + raise + + async def fetchrow(self, sql): + assert sql == "SELECT epoch, deployment_id FROM loom.authority" + await self.step("deployment") + return {"deployment_id": "control-deployment", "epoch": 7} + + async def set_type_codec(self, typ, **kwargs): + assert kwargs["schema"] == "pg_catalog" + await self.step(typ) + + async def close(self, *, timeout): # noqa: ASYNC109 — matches the driver close API. + assert timeout > 0 + self.closes += 1 + self.events.append("public close started") + self.close_started.set() + await self.close_allowed.wait() + self.closed = True + self.events.append("public close finished") + + def terminate(self): + self.terminations += 1 + + def is_closed(self): + return self.closed or bool(self.terminations) + + +@pytest.fixture +def opening(monkeypatch): + gate = retirement.RetirementGate() + monkeypatch.setattr(retirement, "_gate", gate) + monkeypatch.setattr(retirement, "_pid", os.getpid()) + monkeypatch.setattr(pg, "_state_pid", os.getpid()) + for name in ("_pools", "_opening", "_closing"): + monkeypatch.setattr(pg, name, weakref.WeakKeyDictionary()) + monkeypatch.setattr(pg, "_expected", ("control-deployment", 7)) + monkeypatch.setattr(provider_process, "_identity", (os.getpid(), UUID(int=776))) + monkeypatch.setattr(settings, "pg_dsn", "postgresql://unused/control") + monkeypatch.setattr(settings, "pg_statement_timeout_ms", 5000) + monkeypatch.setattr(settings, "cell", "restore") + + def make(): + conn = Connection() + owner = cell_control.Owner() + conn.owner = owner + + async def connect(*args, **kwargs): + return conn + + monkeypatch.setattr(asyncpg, "connect", connect) + return owner, conn + + yield make + assert gate.active == 0 + + +@pytest.mark.parametrize("error_stage", [None, "deployment", "json"]) +async def test_control_owner_binds_before_io_and_closes_initialization_failure( + opening, monkeypatch, error_stage +): + _, conn = opening() + owner = cell_control.Owner() + conn.owner = owner + conn.error_stage = error_stage + for name in ("direct_refusals", "encoding_refusals"): + monkeypatch.setattr(cell_control.apg, name, AsyncMock(return_value=[])) + monkeypatch.setattr(cell_control.apg, "take_lock", AsyncMock()) + try: + if error_stage is None: + await owner.call(lambda: owner._open("update"), total_s=2) + assert not owner.closed and conn.closes == 0 + else: + with pytest.raises(ValueError) as error: + await owner.call(lambda: owner._open("update"), total_s=2) + assert error.value is conn.failure + assert owner.closed and conn.closed and conn.closes == 1 + stages = ["deployment", "json", "jsonb"] + if error_stage is not None: + stages = stages[: stages.index(error_stage) + 1] + assert conn.io == [(stage, True, True) for stage in stages] + assert owner.conn is conn and conn.terminations == 0 + finally: + await owner.close() + assert conn.closed and conn.closes == 1 and conn.terminations == 0 + + +@pytest.mark.parametrize("state", ["closed", "bound"]) +async def test_control_owner_refuses_reuse_before_driver_acquisition(opening, monkeypatch, state): + _, conn = opening() + owner = cell_control.Owner() + owner.closed = state == "closed" + owner.conn = conn if state == "bound" else None + connect = AsyncMock() + monkeypatch.setattr(asyncpg, "connect", connect) + try: + with pytest.raises(cell_control.Refused): + await owner.connect(statement_timeout_ms=5000) + connect.assert_not_awaited() + assert owner.conn is (conn if state == "bound" else None) + finally: + await owner.close() + + +async def control_outcome(owner): + try: + await owner.call(lambda: owner._open("update"), total_s=0.8) + except asyncio.CancelledError as error: + return error + return None + + +@pytest.mark.parametrize("stage", ["deployment", "json"]) +@pytest.mark.parametrize("ending", ["deadline", "cancel"]) +async def test_control_public_close_precedes_initialization_cancellation(opening, stage, ending): + owner, conn = opening() + conn.block_stage = stage + conn.close_allowed.clear() + task = asyncio.create_task(control_outcome(owner)) + closing = asyncio.create_task(conn.close_started.wait()) + try: + await asyncio.wait_for(conn.started.wait(), 1) + if ending == "cancel": + task.cancel("first cancellation") + await asyncio.wait({task, closing}, timeout=1, return_when=asyncio.FIRST_COMPLETED) + assert conn.close_started.is_set(), "initialization must remain owned by public close" + assert "initialization cancelled" not in conn.events + conn.close_allowed.set() + if ending == "deadline": + with pytest.raises(TimeoutError, match="phase deadline expired"): + await asyncio.wait_for(asyncio.shield(task), 1) + else: + error = await asyncio.wait_for(asyncio.shield(task), 1) + assert isinstance(error, asyncio.CancelledError) + assert error.args == ("first cancellation",) + assert conn.events == [ + "public close started", + "public close finished", + "initialization cancelled", + ] + assert conn.closed and conn.closes == 1 and conn.terminations == 0 + finally: + conn.close_allowed.set() + conn.release.set() + for pending in (task, closing): + if not pending.done(): + pending.cancel() + await asyncio.gather(task, closing, return_exceptions=True) + + +async def test_throwing_bind_keeps_raw_cleanup_with_connector(opening): + owner, conn = opening() + failure = ValueError("binding refused") + + def bind(raw): + assert raw is conn + raise failure + + with pytest.raises(ValueError) as error: + await pg._connect_dedicated(bind=bind) + assert error.value is failure + assert owner.conn is None + assert conn.io == [] and conn.closes == 0 and conn.terminations == 1 + + +async def test_placement_uses_owned_connector_and_initializes_once(opening, monkeypatch): + owner, conn = opening() + stop = ValueError("stop before placement work") + monkeypatch.setattr(cell_place, "Owner", lambda **kwargs: owner) + monkeypatch.setattr(cell_place.apg, "direct_refusals", AsyncMock(side_effect=stop)) + with pytest.raises(ValueError) as error: + await cell_place.place( + argparse.Namespace(cell="control", org="acme"), httpx.URL("https://router.test") + ) + assert error.value is stop + assert conn.io == [("deployment", True, True), ("json", True, True), ("jsonb", True, True)] + assert owner.conn is conn and owner.closed + assert conn.closed and conn.closes == 1 and conn.terminations == 0 diff --git a/tests/unit/test_cell_fingerprint_reservations.py b/tests/unit/test_cell_fingerprint_reservations.py new file mode 100644 index 0000000000..461eadd657 --- /dev/null +++ b/tests/unit/test_cell_fingerprint_reservations.py @@ -0,0 +1,183 @@ +"""A cell reserves both accepted values against every other cell's values.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +import pytest + +from server.core import cell + + +def _value(**fields: Any) -> dict[str, Any]: + return cell.as_value(deepcopy(cell.DEFAULTS) | fields) + + +@pytest.mark.parametrize( + "left,right,reason", + [ + ({"bearer_issuers": ["issuer"]}, {"bearer_issuers": ["issuer"]}, "bearer"), + ( + {"oidc_issuer": "https://login/", "oidc_client_id": "client"}, + {"oidc_issuer": "https://login", "oidc_client_id": "client"}, + "OIDC", + ), + ({"end_user_orgs": ["team-*"]}, {"end_user_orgs": ["team-one*"]}, "pattern"), + ({"end_user_orgs": ["team-*"]}, {"configured_orgs": ["team-one"]}, "configured"), + ({"configured_orgs": ["team-one"]}, {"end_user_orgs": ["team-*"]}, "configured"), + ({"configured_orgs": ["same"]}, {"configured_orgs": ["same"]}, "configured"), + ], +) +def test_conflicts_include_next_and_both_directions( + left: dict[str, Any], right: dict[str, Any], reason: str +) -> None: + from server.router.fingerprints import reservation_refusal + + assert reason in reservation_refusal([_value(), _value(**left)], [_value(**right)]) + assert reason in reservation_refusal([_value(**left)], [_value(), _value(**right)]) + assert reason in reservation_refusal([_value(**right)], [_value(**left)]) + + +def test_configured_orgs_are_literal_and_empty_oidc_pair_is_unreserved() -> None: + from server.router.fingerprints import reservation_refusal + + assert ( + reservation_refusal( + [_value(configured_orgs=["team*"])], [_value(configured_orgs=["teams"])] + ) + == "" + ) + assert ( + reservation_refusal([_value(configured_orgs=["team*"])], [_value(end_user_orgs=["teams*"])]) + == "" + ) + assert reservation_refusal([_value(oidc_issuer="issuer")], [_value(oidc_issuer="issuer")]) == "" + assert ( + reservation_refusal([_value(oidc_client_id="client")], [_value(oidc_client_id="client")]) + == "" + ) + + +@pytest.mark.parametrize( + "value", + [ + {"v": True, "fields": {}}, + {"v": 99, "fields": {}}, + {"v": 1, "fields": {"scim": 0}}, + {"v": 1, "fields": {"bearer_issuers": "issuer"}}, + {"v": 1, "fields": {"configured_orgs": ["*"]}}, + {"v": 1, "fields": {"configured_orgs": ["-"]}}, + {"v": 1, "fields": {"configured_orgs": [""]}}, + {"v": 1, "fields": {"configured_orgs": ["x" * (cell.ORG_MAX_BYTES + 1)]}}, + {"v": 1, "fields": {"end_user_orgs": ["*"]}}, + {"v": 1, "fields": {"public_host": "bad\0host"}}, + {"v": 1, "fields": {"future": {"nested": "\ud800"}}}, + {"v": 1, "fields": {"future": float("nan")}}, + {"v": 1, "fields": {"future": ("hidden\0text",)}}, + {"v": 1, "fields": {"future": {1: "non-string key"}}}, + ], +) +def test_unreadable_reservations_refuse_on_either_side(value: Any) -> None: + from server.router.fingerprints import reservation_refusal + + assert "unreadable" in reservation_refusal([_value()], [value]) + assert "unreadable" in reservation_refusal([value], [_value()]) + + +def test_known_defaults_and_unknown_fields_survive_validation() -> None: + from server.router.fingerprints import reservation_refusal + + old = {"v": 1, "fields": {}} + newer = {"v": 1, "fields": {"future": {"items": [1, True, None]}}} + before = deepcopy(newer) + assert reservation_refusal([old], [newer]) == "" + assert newer == before + + +def test_cycle_and_non_json_unknowns_fail_closed() -> None: + from server.router.fingerprints import reservation_refusal + + cyclic = _value() + cyclic["fields"]["future"] = cyclic + assert "unreadable" in reservation_refusal([_value()], [cyclic]) + assert "unreadable" in reservation_refusal([_value()], [_value(future=object())]) + + +def test_candidate_fits_a_live_placement_with_the_largest_org() -> None: + import json + + from server.router.fingerprints import placement_refusal + from server.router.limits import MAX_BODY_BYTES + + deployment = "11111111-1111-4111-8111-111111111111" + value = _value(future="", deployment_id=deployment) + overhead = max( + len(json.dumps({"org": "", "cell": "first", "fingerprint": value})), + len(json.dumps({"org": "", "deployment_id": deployment, "fingerprint": value})), + ) + room = MAX_BODY_BYTES - overhead - 6 * cell.ORG_MAX_BYTES + value["fields"]["future"] = "x" * room + assert placement_refusal("first", "candidate", value) == "" + value["fields"]["future"] += "x" + reason = placement_refusal("first", "candidate", value) + assert "future" in reason + assert "x" * 20 not in reason + + +def test_live_size_cannot_admit_an_oversized_operator_placement() -> None: + import json + + import httpx + + from server.router.fingerprints import placement_refusal, readable + from server.router.limits import MAX_BODY_BYTES + + deployment = "11111111-1111-4111-8111-111111111111" + value = _value(deployment_id=deployment, oidc_client_id="") + live = {"org": "", "cell": "a", "fingerprint": value} + room = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * room + org = "\x01" * cell.ORG_MAX_BYTES + assert cell.named_org(org) == org and readable(value) + live["org"] = org + operator = {"org": org, "deployment_id": deployment, "fingerprint": value} + assert ( + len(httpx.Request("POST", "http://router.example.test", json=live).content) + <= MAX_BODY_BYTES + ) + assert ( + len(httpx.Request("POST", "http://router.example.test", json=operator).content) + > MAX_BODY_BYTES + ) + assert "placement body" in placement_refusal("a", "candidate", value) + + +@pytest.mark.parametrize( + "cell_name,deployment", + [("a", "11111111-1111-4111-8111-111111111111"), ("a" * 32, "d")], +) +def test_both_placement_envelopes_fit_the_accepted_boundary(cell_name, deployment) -> None: + import json + + import httpx + + from server.router.fingerprints import placement_refusal, readable + from server.router.limits import MAX_BODY_BYTES + + value = _value(deployment_id=deployment, oidc_client_id="") + bodies = [ + {"org": "", "cell": cell_name, "fingerprint": value}, + {"org": "", "deployment_id": deployment, "fingerprint": value}, + ] + room = MAX_BODY_BYTES - max(len(json.dumps(body)) for body in bodies) - 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * room + org = "\x01" * cell.ORG_MAX_BYTES + assert cell.named_org(org) == org and readable(value) and cell.NAME.fullmatch(cell_name) + for body in bodies: + body["org"] = org + assert ( + len(httpx.Request("POST", "http://router.example.test", json=body).content) + <= MAX_BODY_BYTES + ) + assert placement_refusal(cell_name, "candidate", value) == "" diff --git a/tests/unit/test_cell_participant_conflicts.py b/tests/unit/test_cell_participant_conflicts.py new file mode 100644 index 0000000000..52883285ae --- /dev/null +++ b/tests/unit/test_cell_participant_conflicts.py @@ -0,0 +1,39 @@ +"""A newer participant row cannot hide disagreeing replicas at an older version.""" + +from __future__ import annotations + +import copy + +import pytest + +from server.cli import cell_control +from server.cli import cell_update_participants as participants +from tests.unit.test_cell_update_participants import participant + +ORDERS = ((0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0)) +PROOF = "host|boot||namespace|1|3|||cell-retirement-v1" + + +@pytest.mark.parametrize("order", ORDERS, ids=lambda order: "".join(map(str, order))) +@pytest.mark.parametrize("difference", ["state", "proof"]) +def test_higher_version_does_not_hide_equal_version_conflict(order, difference): + rows = [participant(1, proof=PROOF, version=2), participant(1, proof=PROOF, version=1)] + conflict = participant(1, proof=PROOF, version=1) + if difference == "state": + conflict["state"] = "retired" + else: + conflict["host_proof"] = PROOF.replace("|1|3|", "|2|3|") + rows.append(conflict) + with pytest.raises(cell_control.Refused, match="conflicting equal versions"): + participants.latest([rows[index] for index in order]) + + +@pytest.mark.parametrize("order", ORDERS, ids=lambda order: "".join(map(str, order))) +def test_matching_old_duplicates_preserve_actual_newest_row_and_input(order): + older = participant(1, proof=PROOF, version=1) + newer = participant(1, "drained", proof=PROOF, version=2) + rows = [newer, older, copy.deepcopy(older)] + ordered = [rows[index] for index in order] + before = copy.deepcopy(ordered) + assert participants.latest(ordered) == {newer["participant_id"]: newer} + assert ordered == before diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py new file mode 100644 index 0000000000..3a8d153f3e --- /dev/null +++ b/tests/unit/test_cell_place.py @@ -0,0 +1,608 @@ +"""Operator placement keeps the directory winner and bounds its local session.""" + +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import asynccontextmanager +from dataclasses import asdict +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.unit.test_router import _row + +OPERATOR = "lco_" + "7" * 128 + + +@pytest.fixture(autouse=True) +def _code_org_reader(monkeypatch): + monkeypatch.setattr(cell, "_code_orgs", None) + + +class DirectoryConnection: + def __init__(self, winner: str | None = "eu-1") -> None: + self.winner = winner + self.row = _row() + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + self.committed = False + + @asynccontextmanager + async def transaction(self): + yield + self.committed = True + + async def execute(self, sql, *args): + self.calls.append((sql, args)) + + async def fetchrow(self, sql, *args): + self.calls.append((sql, args)) + if "FROM loom_router.cells WHERE" in sql: + return asdict(self.row) + if self.winner is not None: + return {"cell": self.winner, "deployment_id": f"dep-{self.winner}"} + return None + + async def fetchval(self, sql, *args): + self.calls.append((sql, args)) + self.winner = args[1] + return args[0] + + +@pytest.fixture +def connected(monkeypatch): + conn = DirectoryConnection() + + @asynccontextmanager + async def connection(**kw): + yield conn + + monkeypatch.setattr(directory, "_connection", connection) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + return conn + + +def request(client, **over): + return client.post( + "/cells/v1/operator/cells/eu-1/place", + json={ + "org": "acme", + "deployment_id": "dep-eu-1", + "fingerprint": cell.as_value( + {"public_host": "eu-1.example.test", "deployment_id": "dep-eu-1"} + ), + **over, + }, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + +def test_existing_winner_emits_audit_wal_before_returning(connected): + client = TestClient(app.build_app(app.Router())) + result = request(client) + assert result.status_code == 200 + assert result.json() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + audit = [(sql, args) for sql, args in connected.calls if "INSERT INTO loom_router.audit" in sql] + assert len(audit) == 1 and audit[0][1] == ("operator", {"org": "acme", "cell": "eu-1"}) + assert connected.committed + assert not any("UPDATE loom_router.cells SET placed" in sql for sql, _ in connected.calls) + + +@pytest.mark.parametrize( + "posted", + [{"deployment_id": "other"}, {"fingerprint": {}}, {"org": "x" * (cell.ORG_MAX_BYTES + 1)}], +) +def test_existing_winner_refuses_operator_identity_mismatch(connected, posted): + result = request(TestClient(app.build_app(app.Router())), **posted) + assert result.status_code in (400, 409) + assert not connected.committed + assert not any("INSERT" in sql for sql, _ in connected.calls) + + +def test_operator_authentication_precedes_directory_access(connected): + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", json={} + ) + assert result.status_code == 401 and connected.calls == [] + + +def test_live_duplicate_also_emits_audit_wal(connected): + assert asyncio.run( + directory.place("acme", "eu-1", lambda row: "closed", actor="cell:eu-1") + ) == ("eu-1", "dep-eu-1") + assert connected.committed + assert any("INSERT INTO loom_router.audit" in sql for sql, _ in connected.calls) + + +class LocalConnection: + def __init__(self): + self.calls = [] + self.committed = False + self.terminated = False + self.closed = False + self.stall_commit = False + self.stall_close = False + self.fail_insert = False + self.commit_started = asyncio.Event() + + def transaction(self): + conn = self + + class Transaction: + async def start(self): + pass + + async def commit(self): + conn.commit_started.set() + if conn.stall_commit: + await asyncio.Event().wait() + conn.committed = True + + return Transaction() + + async def execute(self, sql, *args): + self.calls.append(sql) + if "INSERT" in sql and self.fail_insert: + raise OSError("local insert failed") + return "INSERT 0 0" + + async def close(self, **kwargs): + self.calls.append("close") + if self.stall_close: + await asyncio.Event().wait() + self.closed = True + + def terminate(self): + self.terminated = True + + def is_closed(self): + return self.closed or self.terminated + + +@pytest.fixture +def local(monkeypatch): + import asyncpg + + from server.cli import cell_place + from server.core import pg + + monkeypatch.setattr(settings, "pg_dsn", "postgres://not-connected") + monkeypatch.setattr(pg, "_expected", None) + conn = LocalConnection() + state = {"first": "eu-1", "state": "enforced", "deployment": "dep-eu-1", "row": None} + + async def read_state(c): + return state["state"], state["first"] + + async def fetchval(sql): + return state["deployment"] + + async def fetchrow(sql): + return state["row"] + + conn.fetchval = fetchval + conn.fetchrow = fetchrow + + async def connect(*args, **driver_kw): + kw = {"statement_timeout_ms": int(driver_kw["server_settings"]["statement_timeout"])} + assert kw == {"statement_timeout_ms": settings.pg_statement_timeout_ms} + return conn + + async def initialize(c): + assert c is conn + + async def direct(c, what): + return [] + + async def take_lock(c, what): + conn.calls.append("lock") + + async def held(c): + return True + + async def directory_call(*a): + conn.calls.append("directory") + + monkeypatch.setattr(asyncpg, "connect", connect) + monkeypatch.setattr(pg, "_check_deployment", initialize) + monkeypatch.setattr(pg, "init_connection", initialize) + monkeypatch.setattr(cell, "read_state_if_present", read_state) + monkeypatch.setattr(cell_place.apg, "direct_refusals", direct) + monkeypatch.setattr(cell_place.apg, "take_lock", take_lock) + monkeypatch.setattr(cell_place.apg, "lock_held", held) + monkeypatch.setattr(cell_place, "_directory", directory_call) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.03) + fields = cell.fingerprint("dep-eu-1") + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + return conn, state + + +async def invoke_place(): + import argparse + + import httpx + + from server.cli import cell_place + + return await cell_place.place( + argparse.Namespace(cell="eu-1", org="acme"), httpx.URL("https://router.test") + ) + + +async def test_local_noop_insert_still_commits_durability_message(local): + from server.core import pg + + conn, state = local + assert await invoke_place() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + assert conn.committed and conn.closed and not conn.terminated + assert conn.calls[0:2] == ["lock", "directory"] + assert conn.calls[-2:] == [pg._DURABLE_MESSAGE, "close"] + + +@pytest.mark.parametrize( + "change", + [ + {"state": "recording"}, + {"state": "partial"}, + {"first": "us-1"}, + {"deployment": "other"}, + {"row": None}, + ], +) +async def test_local_invalid_identity_refuses_before_directory(local, change): + from server.cli import cell_place + + conn, state = local + state.update(change) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert conn.calls == ["lock", "close"] and conn.closed and not conn.terminated + + +@pytest.mark.parametrize("change", [{"state": "recording"}, {"first": "us-1"}, {"row": None}]) +async def test_local_identity_is_read_again_before_insert(local, monkeypatch, change): + from server.cli import cell_place + + conn, state = local + + async def changed(*a): + state.update(change) + + monkeypatch.setattr(cell_place, "_directory", changed) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert not any("INSERT" in sql for sql in conn.calls) + assert conn.closed and not conn.terminated and not conn.committed + + +async def test_local_insert_failure_can_be_retried(local): + conn, state = local + conn.fail_insert = True + with pytest.raises(OSError, match="local insert failed"): + await invoke_place() + assert not conn.committed and conn.closed and not conn.terminated + conn.fail_insert = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_and_cleanup_stalls_are_bounded(local): + conn, state = local + conn.stall_commit = conn.stall_close = True + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert conn.commit_started.is_set() and conn.terminated and not conn.committed + conn.stall_commit = conn.stall_close = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_cancellation_closes_the_session(local): + conn, state = local + conn.stall_commit = True + task = asyncio.create_task(invoke_place()) + try: + try: + await asyncio.wait_for(conn.commit_started.wait(), 0.5) + except TimeoutError: + if task.done(): + await task + raise + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 0.5) + assert conn.closed and not conn.terminated and not conn.committed + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): + import asyncpg + + conn, state = local + cleaned = False + + async def stalled(*args, **kw): + nonlocal cleaned + try: + await asyncio.Event().wait() + finally: + cleaned = True + + monkeypatch.setattr(asyncpg, "connect", stalled) + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert cleaned and conn.calls == [] + + +async def test_query_timeout_cannot_enter_an_unbounded_rollback(local): + conn, state = local + + class Transaction: + async def start(self): + pass + + async def commit(self): + pytest.fail("a stalled statement cannot commit") + + async def __aenter__(self): + await self.start() + + async def __aexit__(self, *exc): + await asyncio.Event().wait() + + async def stalled(sql, *args): + await asyncio.Event().wait() + + conn.transaction = Transaction + conn.execute = stalled + task = asyncio.create_task(invoke_place()) + try: + done, _ = await asyncio.wait([task], timeout=0.15) + assert done, "the statement deadline must include ending its transaction" + with pytest.raises(TimeoutError): + await task + assert conn.closed and not conn.terminated + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize( + "status,answer", + [ + (200, {"org": "acme", "cell": "us-1", "deployment_id": "dep-us-1"}), + (200, {"org": "acme", "cell": "eu-1", "deployment_id": "wrong"}), + (200, []), + (200, {"org": "someone-else", "cell": "eu-1", "deployment_id": "dep-eu-1"}), + (503, {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"}), + (504, {"error": "gateway timeout"}), + ], +) +async def test_cli_refuses_unknown_response_or_other_winner(monkeypatch, status, answer): + import httpx + + from server.cli import cell_place + + async def respond(request): + assert request.headers["authorization"] == f"Bearer {settings.router_operator_token}" + return httpx.Response(status, json=answer) + + real = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real(transport=httpx.MockTransport(respond), **kw) + ) + with pytest.raises(cell_place.PlacementRefused): + await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +async def test_repository_only_configured_orgs_are_in_full_cli_fingerprint(local, monkeypatch): + from server.cli import cell_place + from server.code.domain.index import repos + + conn, state = local + monkeypatch.setattr(repos, "operator_orgs", lambda: {"repo-only"}) + fields = cell.fingerprint("dep-eu-1") + fields["configured_orgs"] = sorted(set(fields["configured_orgs"]) | {"repo-only"}) + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + _, value = await cell_place._identity(conn, "eu-1") + assert "repo-only" in value["fields"]["configured_orgs"] + + +def test_new_placement_uses_counted_statement_bound(monkeypatch): + from tests.unit.test_router_attach import _cell_row, _Conn, _Store + + store = _Store(_cell_row(state="serving")) + + @asynccontextmanager + async def connection(**kw): + yield _Conn(store) + + monkeypatch.setattr(directory, "_connection", connection) + asyncio.run( + directory.place("new", "eu-1", lambda row: "", actor="operator", verify=lambda row: "") + ) + assert store.statements == directory.PLACE_STATEMENTS + assert directory.PLACE_MAX_S == directory._bound_s(store.statements) + + +def test_operator_placement_body_bound_refuses_before_directory(connected): + from server.router.limits import MAX_BODY_BYTES + + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", + content=b"x" * (MAX_BODY_BYTES + 1), + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + assert result.status_code == 413 and connected.calls == [] + + +def test_operator_place_bypasses_cell_token_rate_limiter(connected, monkeypatch): + router = app.Router() + monkeypatch.setattr( + router.placements, "admit", lambda name: pytest.fail("operator used cell rate") + ) + assert request(TestClient(app.build_app(router))).status_code == 200 + + +@pytest.mark.parametrize("org", ["", "-", "*", "bad\x00org"]) +def test_operator_place_refuses_malformed_org_before_directory(connected, org): + assert request(TestClient(app.build_app(app.Router())), org=org).status_code == 400 + assert connected.calls == [] + + +def test_cli_cancellation_names_the_idempotent_retry_and_preserves_cancellation( + monkeypatch, capsys +): + import argparse + + import httpx + + from server.cli import cell_place + + cancelled = asyncio.CancelledError("commit response was cancelled") + + async def cancel(*args): + raise cancelled + + monkeypatch.setattr(cell_place, "_args_problem", lambda args: "") + monkeypatch.setattr( + cell_place.cell_cli, "operator_router", lambda args: httpx.URL("https://router.test") + ) + monkeypatch.setattr(cell_place, "place", cancel) + monkeypatch.setattr(cell_place, "run_connected", lambda run, **kw: asyncio.run(run(None))) + with pytest.raises(asyncio.CancelledError) as caught: + cell_place.cmd_place(argparse.Namespace(cell="eu-1", org="acme")) + assert caught.value is cancelled + printed = capsys.readouterr() + assert printed.out == "" + assert "placement may have committed; rerun the same command" in printed.err + + +async def test_local_phases_remain_separate_from_directory_and_exit(local, monkeypatch): + from server.cli import cell_control, cell_place + + observed = [] + actual_call = cell_control.Owner.call + + async def timed(self, work, *, total_s): + observed.append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def directory_call(*args): + # The complete HTTP exchange has its independent directory-derived phase. + await asyncio.sleep(0.06) + + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(cell_place, "_directory", directory_call) + await invoke_place() + assert observed == [0.03, 0.03, cell_place.REQUEST_S, 0.03] + conn, _ = local + assert conn.committed and conn.closed and not conn.terminated + + +@pytest.mark.parametrize("where", ["initialization", "initial", "transaction", "directory"]) +async def test_original_error_survives_failed_required_close(local, monkeypatch, where): + from server.cli import cell_place + from server.core import pg + + original = ValueError("original placement failure") + conn, _ = local + + async def fail(*args): + raise original + + async def bad_close(**kwargs): + raise RuntimeError("close failed") + + conn.close = bad_close + if where == "initialization": + monkeypatch.setattr(pg, "init_connection", fail) + elif where == "initial": + monkeypatch.setattr(cell_place, "_identity", fail) + elif where == "directory": + monkeypatch.setattr(cell_place, "_directory", fail) + else: + conn.execute = fail + with pytest.raises(ValueError) as caught: + await invoke_place() + assert caught.value is original + assert conn.terminated and not conn.committed + + +async def test_success_refuses_when_required_close_fails(local): + conn, _ = local + + async def fail(**kwargs): + raise OSError("required close failed") + + conn.close = fail + with pytest.raises(OSError, match="required close failed"): + await invoke_place() + assert conn.committed and conn.terminated + + +async def test_lost_original_lock_refuses_without_reconnecting(local, monkeypatch): + from server.cli import cell_place + + conn, _ = local + + async def lost(c): + assert c is conn + return False + + monkeypatch.setattr(cell_place.apg, "lock_held", lost) + with pytest.raises(cell_place.PlacementRefused, match="lock was lost"): + await invoke_place() + assert conn.calls == ["lock", "directory", "close"] + assert conn.closed and not conn.committed + + +@pytest.mark.parametrize("inside_reserve", [False, True]) +async def test_late_placement_connection_never_reaches_body(local, monkeypatch, inside_reserve): + import asyncpg + + conn, _ = local + released, discarded = asyncio.Event(), asyncio.Event() + real_terminate = conn.terminate + real_close = conn.close + + def terminate(): + real_terminate() + discarded.set() + + async def close(**kw): + await real_close(**kw) + discarded.set() + + async def late(*args, **kwargs): + try: + await asyncio.Future() + except asyncio.CancelledError: + if not inside_reserve: + while not released.is_set(): + try: + await released.wait() + except asyncio.CancelledError: + pass + return conn + + conn.terminate, conn.close = terminate, close + monkeypatch.setattr(asyncpg, "connect", late) + with pytest.raises(TimeoutError): + await invoke_place() + released.set() + await asyncio.wait_for(discarded.wait(), 0.2) + assert conn.calls == (["close"] if inside_reserve else []) + assert conn.closed if inside_reserve else conn.terminated + assert not conn.committed diff --git a/tests/unit/test_cell_place_envelopes.py b/tests/unit/test_cell_place_envelopes.py new file mode 100644 index 0000000000..40b89fabda --- /dev/null +++ b/tests/unit/test_cell_place_envelopes.py @@ -0,0 +1,332 @@ +"""Every accepted fingerprint fits both placement envelopes for every valid org.""" + +from __future__ import annotations + +import hashlib +import itertools +import json +from types import SimpleNamespace +from uuid import UUID + +import httpx +import pytest +from starlette.requests import Request + +from server.core import cell +from server.router import app, directory +from server.router.limits import MAX_BODY_BYTES +from server.settings import settings + +DEPLOYMENT = str(UUID(int=1)) +TOKEN = "lco_" + "7" * 128 + + +def fingerprint(): + return cell.as_value( + { + **cell.DEFAULTS, + "deployment_id": DEPLOYMENT, + "public_host": "first.example.test", + "oauth_issuer_host": "first.example.test", + } + ) + + +def envelopes(name, value, org=""): + return ( + {"org": org, "cell": name, "fingerprint": value}, + {"org": org, "deployment_id": DEPLOYMENT, "fingerprint": value}, + ) + + +def attaching(name): + row = SimpleNamespace( + cell=name, + is_first=True, + state="attaching", + mode="recording", + deployment_id=DEPLOYMENT, + current_fp=None, + next_fp=None, + host="first.example.test", + ) + return SimpleNamespace(row=row, other=None, attach_started_at=None, bucket="", bucket_prefix="") + + +def begin(value, position): + return directory.Begin( + DEPLOYMENT, + value if position == "current_fp" else fingerprint(), + value if position == "next_fp" else None, + "a" * 64, + {}, + ) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_reserves_operator_envelope_before_accepting_fingerprint(name, position): + value = fingerprint() + live = envelopes(name, value)[0] + # This fits the previous live-only reservation, but not the operator envelope. + live_size = len(json.dumps(live)) + 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - live_size) + reason = directory.begin_refusal(attaching(name), begin(value, position)) + assert position in reason and str(MAX_BODY_BYTES) in reason + assert "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("size", ["default", "boundary"]) +@pytest.mark.parametrize("org", ["acme", "\x01" * cell.ORG_MAX_BYTES]) +async def test_accepted_fingerprint_fits_both_encodings_and_operator_handler( + monkeypatch, name, position, size, org +): + value = fingerprint() + if size == "boundary": + empty_size = max(len(json.dumps(body)) for body in envelopes(name, value)) + value["fields"]["oidc_client_id"] = "x" * ( + MAX_BODY_BYTES - empty_size - 6 * cell.ORG_MAX_BYTES + ) + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + attempt = begin(value, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + bodies = envelopes(name, value, org) + encoded = [ + (json.dumps(body).encode(), httpx.Request("POST", "http://inert.test", json=body).content) + for body in bodies + ] + assert all(len(raw) <= MAX_BODY_BYTES for pair in encoded for raw in pair) + calls = [] + + async def place(actual_org, actual_name, refusal, *, actor, verify): + calls.append((actual_org, actual_name, actor)) + locked = SimpleNamespace( + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + ) + assert verify(locked) == "" + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + for raw in encoded[1]: + scope = { + "type": "http", + "method": "POST", + "path": f"/cells/v1/operator/cells/{name}/place", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(raw=raw): + return {"type": "http.request", "body": raw, "more_body": False} + + response = await app.Router().operator_place(Request(scope, receive)) + assert response.status_code == 200 + assert json.loads(response.body) == {"org": org, "cell": name, "deployment_id": DEPLOYMENT} + assert calls == [(org, name, "operator")] * 2 + + +OMITTABLE_DEFAULTS = ( + "bearer_issuers", + "bucket", + "bucket_prefix", + "configured_orgs", + "end_user_orgs", + "oidc_issuer", + "scim", +) + + +@pytest.fixture +def native_fingerprint(monkeypatch): + configured = SimpleNamespace( + public_base_url="https://first.example.test", + oauth_issuer="", + oidc_issuer="", + oidc_client_id="", + scim_token="", + document_s3_bucket="", + document_s3_prefix="", + oidc_jit_org="", + self_signup="closed", + self_signup_org="", + code_org="", + tracker_org="", + trusted_issuers=lambda: set(), + end_user_issuers=lambda: {}, + ) + monkeypatch.setattr(cell, "settings", configured) + monkeypatch.setattr(cell, "_code_orgs", None) + + def native(client_id): + configured.oidc_client_id = client_id + # Both native placement callers serialize this complete field set. + return cell.as_value(cell.fingerprint(DEPLOYMENT)) + + return native + + +def omitted_values(): + return ( + subset + for count in range(len(OMITTABLE_DEFAULTS) + 1) + for subset in itertools.combinations(OMITTABLE_DEFAULTS, count) + ) + + +def stored_fingerprint(native, omitted, unknown): + return cell.as_value( + { + **{key: value for key, value in native["fields"].items() if key not in omitted}, + **unknown, + } + ) + + +def accepted_row(name, attempt): + return SimpleNamespace( + cell=name, + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + state="serving", + is_first=True, + mode="enforced", + closed=False, + holds=lambda token: True, + ) + + +async def route_response(monkeypatch, name, attempt, body, operator): + calls = [] + row = accepted_row(name, attempt) + + async def place(org, actual_name, refusal, *, actor, verify=None): + assert actual_name == name and refusal(row) == "" + if verify is not None: + assert verify(row) == "" + calls.append((org, actor)) + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + router = app.Router() + + async def authenticated(request, actual_name): + assert actual_name == name + return "inert-cell-credential" + + monkeypatch.setattr(router, "_authenticated", authenticated) + monkeypatch.setattr(router, "placements", SimpleNamespace(admit=lambda name: True)) + raw = httpx.Request("POST", "http://inert.test", json=body).content + scope = { + "type": "http", + "method": "POST", + "path": "/cells/v1/placements", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(): + return {"type": "http.request", "body": raw, "more_body": False} + + handler = router.operator_place if operator else router.place + response = await handler(Request(scope, receive)) + return response, calls, len(raw) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x"}, {"future": "x" * 512}]) +def test_sparse_admission_reserves_matching_native_callers( + native_fingerprint, name, position, unknown +): + org = "\x01" * cell.ORG_MAX_BYTES + for omitted in omitted_values(): + stored = stored_fingerprint(native_fingerprint(""), omitted, unknown) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + stored["fields"]["oidc_client_id"] = client_id + native = native_fingerprint(client_id) + attempt = begin(stored, position) + row = accepted_row(name, attempt) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(row, native) + reason = directory.begin_refusal(attaching(name), attempt) + if not reason: + # Admission must cover what these callers send, even if storage is sparse. + assert all( + len(httpx.Request("POST", "http://inert.test", json=body).content) <= MAX_BODY_BYTES + for body in envelopes(name, native, org) + ), (omitted, unknown) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +async def test_sparse_boundary_refuses_unusable_native_requests( + monkeypatch, native_fingerprint, name, position, operator +): + stored = stored_fingerprint(native_fingerprint(""), OMITTABLE_DEFAULTS, {}) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + stored["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(stored["fields"]["oidc_client_id"]) + attempt = begin(stored, position) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + body = envelopes(name, native, org)[int(operator)] + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + if org == "acme": + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + else: + assert size > MAX_BODY_BYTES + assert response.status_code == (413 if operator else 400) + assert not calls + reason = directory.begin_refusal(attaching(name), attempt) + assert position in reason and "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x" * 512}]) +async def test_accepted_sparse_boundary_serves_native_requests( + monkeypatch, native_fingerprint, name, position, operator, unknown +): + native = native_fingerprint("") + # Unknown stored fields keep their existing reservation, although native callers + # do not emit them. Reserve that representation with all known defaults present. + reserved = cell.as_value({**native["fields"], **unknown}) + empty = max(len(json.dumps(body)) for body in envelopes(name, reserved)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(client_id) + stored = stored_fingerprint(native, OMITTABLE_DEFAULTS, unknown) + attempt = begin(stored, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + body = envelopes(name, native, org)[int(operator)] + assert len(json.dumps(body).encode()) <= MAX_BODY_BYTES + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + stored["fields"]["future"] = "x" * MAX_BODY_BYTES + assert position in directory.begin_refusal(attaching(name), begin(stored, position)) diff --git a/tests/unit/test_cell_place_http_deadline.py b/tests/unit/test_cell_place_http_deadline.py new file mode 100644 index 0000000000..79d278903a --- /dev/null +++ b/tests/unit/test_cell_place_http_deadline.py @@ -0,0 +1,157 @@ +"""A complete HTTP exchange cannot outlive its directory-derived owner phase.""" + +from __future__ import annotations + +import asyncio +import json +import time + +import httpcore +import httpx +import pytest +from httpcore._backends.mock import AsyncMockBackend, AsyncMockStream + +from server.cli import cell_control, cell_place +from server.settings import settings +from tests.unit import test_cell_place as helpers + +local = helpers.local +invoke_place = helpers.invoke_place + +_DIRECTORY = cell_place._directory + + +@pytest.fixture +def exchange(local, monkeypatch): + conn, _ = local + events = [] + entered = asyncio.Event() + state = {"mode": "response", "reads": [], "owners": []} + real_close = conn.close + real_call = cell_control.Owner.call + actual_client = httpx.AsyncClient + + async def close(**kwargs): + events.append("pg-close") + await asyncio.sleep(0) + await real_close(**kwargs) + events.append("pg-ack") + + async def timed(owner, work, *, total_s): + if total_s == 0.1: + state["owners"].append(owner) + return await real_call(owner, work, total_s=total_s) + + class Stream(AsyncMockStream): + def __init__(self): + super().__init__([]) + self.headers = False + + async def read(self, max_bytes, timeout=None): # noqa: ASYNC109 — httpcore transport API + if not self.headers: + self.headers = True + if state["mode"] == "response": + return b"HTTP/1.1 200 OK\r\nContent-Length: 100000000\r\n\r\n" + status = b"503 Unavailable" if state["mode"] == "refusal" else b"200 OK" + body = json.dumps( + {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"} + if state["mode"] == "refusal" + else {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + ).encode() + # A timely reply may exceed the separate local phase allowance. + await asyncio.sleep(0.04) + return ( + b"HTTP/1.1 " + + status + + b"\r\nContent-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + entered.set() + state["reads"].append(timeout) + try: + async with asyncio.timeout(timeout): + await asyncio.sleep(0.002) + return b" " + except asyncio.CancelledError: + events.append("http-cancel") + raise + + class Backend(AsyncMockBackend): + async def connect_tcp(self, *args, **kwargs): + return Stream() + + class Client(actual_client): + async def __aexit__(self, *args): + if state["mode"] == "client-exit": + entered.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + events.append("http-cancel") + raise + return await super().__aexit__(*args) + + def client(**kwargs): + transport = httpx.AsyncHTTPTransport() + transport._pool = httpcore.AsyncConnectionPool(network_backend=Backend([])) + return Client(transport=transport, **kwargs) + + conn.close = close + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "REQUEST_S", 0.1, raising=False) + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return conn, events, entered, state + + +@pytest.mark.parametrize("blocking", ["response", "client-exit"]) +@pytest.mark.parametrize("stop", ["deadline", "cancel"]) +async def test_whole_exchange_closes_pg_before_http_cancellation(exchange, blocking, stop): + conn, events, entered, state = exchange + state["mode"] = blocking + started = time.monotonic() + task = asyncio.create_task(invoke_place()) + try: + await asyncio.wait_for(entered.wait(), 0.2) + if stop == "cancel": + task.cancel("operator stop") + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "per-read HTTP timeouts cannot bound a continuing response or client cleanup" + with pytest.raises( + TimeoutError if stop == "deadline" else asyncio.CancelledError + ) as caught: + await task + if stop == "cancel": + assert str(caught.value) == "operator stop" + assert time.monotonic() - started < 0.3 + assert conn.closed and not conn.committed and not conn.terminated + assert not any("INSERT" in call for call in conn.calls) + assert events.index("pg-ack") < events.index("http-cancel") + assert events.count("pg-close") == 1 + if stop == "deadline": + (owner,) = state["owners"] + assert owner.close_end == owner.phase_end + if blocking == "response": + assert state["reads"] and all(value == 0.1 for value in state["reads"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("mode", ["success", "refusal"]) +async def test_timely_actual_http_reply_keeps_local_phase_bounds(exchange, mode): + conn, events, _, state = exchange + state["mode"] = mode + if mode == "refusal": + with pytest.raises(cell_place.PlacementRefused, match="standby stalled"): + await invoke_place() + assert not conn.committed + else: + assert (await invoke_place())["deployment_id"] == "dep-eu-1" + assert conn.committed + assert conn.closed and not conn.terminated + assert events.count("pg-close") == 1 diff --git a/tests/unit/test_cell_place_http_first_error.py b/tests/unit/test_cell_place_http_first_error.py new file mode 100644 index 0000000000..b8120702c9 --- /dev/null +++ b/tests/unit/test_cell_place_http_first_error.py @@ -0,0 +1,164 @@ +"""Placement HTTP failures keep their outcome through client and Postgres cleanup.""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from server.cli import cell_place +from tests.unit import test_cell_native_http_first_error as native +from tests.unit import test_cell_place as place_helpers + +local = place_helpers.local +invoke_place = place_helpers.invoke_place +exchange = native.exchange +observed = native.observed +assert_first = native.assert_first +assert_exit_first = native.assert_exit_first +assert_close_failure = native.assert_close_failure +FIRST_ERRORS = native.FIRST_ERRORS +EXIT_ERRORS = native.EXIT_ERRORS +_BOUNDARIES = ("placement",) +_DIRECTORY = cell_place._directory + + +async def invoke(boundary): + return await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + assert result is None + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_keeps_http_first_failure_and_closes_pg_once( + local, exchange, monkeypatch, first_kind, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") + kind, result = await observed(invoke_place) + assert kind == "error" + if first_kind is asyncio.CancelledError: + # A cancelled child Task reconstructs this exception when Owner reads it. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_required_http_close_failure_prevents_insert( + local, exchange, monkeypatch, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.second = second_kind("required-exit") + kind, result = await observed(invoke_place) + assert kind == "error" + if second_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.second + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +async def test_owned_placement_successful_http_neighbor_commits(local, exchange, monkeypatch): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + assert await invoke_place() == exchange.answer + assert conn.calls.count("close") == 1 + assert conn.closed and conn.committed and not conn.terminated diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py index 14839797c2..a9728ee609 100644 --- a/tests/unit/test_cell_place_owner.py +++ b/tests/unit/test_cell_place_owner.py @@ -178,6 +178,83 @@ def is_closed(self): await asyncio.gather(*owned_tasks, return_exceptions=True) +@pytest.mark.parametrize("stage", ["opening", "initial", "directory", "transaction", "close"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +async def test_placement_fatal_stays_inside_owner_until_public_close(stage, kind): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse, asyncio, sys + import asyncpg + import httpx + from server.cli import cell_place + from server.core import pg + from server.settings import settings + settings.pg_dsn = "postgres://not-connected" + stage, kind = sys.argv[1:] + original = (SystemExit if kind == 'SystemExit' else KeyboardInterrupt)('original') + events = [] + class Transaction: + async def start(self): pass + async def commit(self): events.append('commit') + class Connection: + def transaction(self): return Transaction() + async def execute(self, *args): + if stage == 'transaction': raise original + async def close(self, **kw): + events.append('close') + if stage == 'close': raise original + def terminate(self): events.append('abort') + conn = Connection() + async def connect(*args, **kw): return conn + async def initialize(c): + assert c is conn + if stage == 'opening': raise original + async def noop(*args): pass + async def direct(c, what): return [] + async def identity(c, name): + assert c is conn + if stage == 'initial': raise original + return 'dep', {} + async def directory(*args): + if stage == 'directory': raise original + async def held(c): return True + async def main(): + asyncpg.connect = connect + pg._check_deployment = initialize + pg.init_connection = noop + cell_place.apg.direct_refusals = direct + cell_place.apg.take_lock = noop + cell_place.apg.lock_held = held + cell_place._identity = identity + cell_place._directory = directory + try: + await cell_place.place(argparse.Namespace(cell='first', org='acme'), httpx.URL('http://router')) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 1, events + await asyncio.sleep(0) + print('original-after-public-close') + return + raise AssertionError('fatal became success') + asyncio.run(main()) + """ + ) + result = await asyncio.to_thread( + subprocess.run, + [sys.executable, "-c", program, stage, kind], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-public-close" in result.stdout + + async def test_repeated_signals_cannot_extend_an_expired_public_close(monkeypatch): clock = ManualClock(t=100.0) local_time = SimpleNamespace(monotonic=clock.monotonic) diff --git a/tests/unit/test_cell_update.py b/tests/unit/test_cell_update.py new file mode 100644 index 0000000000..0de63c3f8f --- /dev/null +++ b/tests/unit/test_cell_update.py @@ -0,0 +1,147 @@ +"""The update control command validates its action before connecting or admitting work.""" + +from __future__ import annotations + +import argparse +import importlib + +import pytest + +from server.settings import settings + + +def args(**changes): + return argparse.Namespace( + cell="first", wait=0, config=None, finish=False, revert=False, probe_url=None, **changes + ) + + +@pytest.mark.parametrize("wait", [-1, float("inf"), float("nan"), 86401]) +def test_update_refuses_invalid_wait_before_connect(monkeypatch, wait): + command = importlib.import_module("server.cli.cell_update") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://configured") + values = args() + values.wait = wait + assert "wait" in command._args_problem(values) + + +@pytest.mark.parametrize( + "actions", + [ + {"config": "config.yaml", "finish": True}, + {"finish": True, "revert": True}, + {"probe_url": "http://probe", "revert": True}, + ], +) +def test_update_refuses_ambiguous_actions_before_connect(monkeypatch, actions): + command = importlib.import_module("server.cli.cell_update") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://configured") + values = args() + for name, value in actions.items(): + setattr(values, name, value) + assert "one" in command._args_problem(values) + + +@pytest.mark.parametrize("action", [None, "config", "finish", "revert"]) +def test_scim_cutoffs_require_a_configuration_action(monkeypatch, action): + command = importlib.import_module("server.cli.cell_update") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://configured") + values = args(revoke_unprovisioned=True) + if action is not None: + setattr(values, action, "target.yaml" if action == "config" else True) + result = command._args_problem(values) + assert result == ( + "--revoke-unprovisioned needs --config, --finish or --revert" if action is None else "" + ) + + +@pytest.mark.parametrize("refused", [False, True]) +def test_control_coordinator_verifies_and_gates_without_becoming_a_participant( + monkeypatch, refused +): + from unittest.mock import AsyncMock + + from server.cli import _ch + + events = [] + client = object() + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + + async def connect(ch): + assert ch is client + events.append("paired-stores") + + async def acceptance(*, org_free): + assert org_free.kind == "write" + events.append("org-free-gate") + if refused: + raise _ch.OrgNotHeld("wrong cell") + + async def body(ch): + assert ch is client + events.append("body") + return 7 + + async def close(ch): + assert ch is client + events.append("disconnect") + + forbidden = AsyncMock(side_effect=AssertionError("update must not enroll")) + monkeypatch.setattr(_ch, "connect_verified", connect) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(_ch, "disconnect", close) + monkeypatch.setattr("server.insights.app.provider_command.model_command", forbidden) + if refused: + with pytest.raises(_ch.OrgNotHeld, match="wrong cell"): + _ch.run_control_connected(body, org_free=_ch.writes("cell control")) + else: + assert _ch.run_control_connected(body, org_free=_ch.writes("cell control")) == 7 + assert events == [ + "paired-stores", + "org-free-gate", + *([] if refused else ["body"]), + "disconnect", + ] + forbidden.assert_not_called() + + +@pytest.mark.parametrize("failure_at", ["acceptance", "body", None]) +def test_control_coordinator_preserves_failure_when_disconnect_fails(monkeypatch, failure_at): + from unittest.mock import AsyncMock + + from server.cli import _ch + + client = object() + first = ValueError("coordinator refused") + closing = RuntimeError("disconnect failed") + closed = [] + + async def acceptance(**kwargs): + if failure_at == "acceptance": + raise first + + async def body(ch): + assert ch is client + if failure_at == "body": + raise first + return 7 + + async def close(ch): + closed.append(ch) + raise closing + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(_ch, "connect_verified", AsyncMock()) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(_ch, "disconnect", close) + expected = first if failure_at is not None else closing + with pytest.raises(type(expected)) as raised: + _ch.run_control_connected(body, org_free=_ch.writes("cell control")) + assert raised.value is expected + assert closed == [client] diff --git a/tests/unit/test_cell_update_config.py b/tests/unit/test_cell_update_config.py new file mode 100644 index 0000000000..28c3af0be8 --- /dev/null +++ b/tests/unit/test_cell_update_config.py @@ -0,0 +1,178 @@ +"""An update calculates its target without changing the running configuration.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from server.code.domain.index import repos +from server.core import cell +from server.settings import Settings, settings +from server.settings._file import SettingsFileError + + +def _config(**values: object) -> Settings: + return Settings(_env_file=None, **(settings.model_dump() | values)) + + +def test_the_overlay_wins_over_old_environment_without_installing_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from server.cli.cell_update_config import read_target + + current = _config(code_org="before", tracker_org="tracker") + monkeypatch.setenv("LOOM_CODE_ORG", "environment") + # A candidate validates supplied values, without rereading another settings file. + monkeypatch.setenv("LOOM_SETTINGS_FILE", str(tmp_path / "absent.yaml")) + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings:\n code_org: after\n") + before_env, before_settings = dict(os.environ), settings.model_dump() + target = read_target(path, current) + assert target.code_org == "after" + assert target.tracker_org == "tracker" + assert current.code_org == "before" + assert dict(os.environ) == before_env + assert settings.model_dump() == before_settings + + +@pytest.mark.parametrize( + "body,field", + [ + (" code_org: one\n code_org: two\n", "code_org"), + (" jwt_secret: do-not-copy-this-value\n", "jwt_secret"), + (" memory_ask_deep_evidence_max_memories: -1\n", "memory_ask_deep_evidence_max_memories"), + ], +) +def test_invalid_overlay_fields_refuse_without_echoing_values( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: str, field: str +) -> None: + from server.cli.cell_update_config import read_target + + current = _config() + monkeypatch.setenv("LOOM_MEMORY_ASK_DEEP_EVIDENCE_MAX_MEMORIES", "5") + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings:\n" + body) + with pytest.raises(SettingsFileError) as exc: + read_target(path, current) + if field == "memory_ask_deep_evidence_max_memories": + assert field in str(exc.value) + else: + assert "invalid target settings file" in str(exc.value) + assert "do-not-copy-this-value" not in str(exc.value) + + +def test_full_model_rules_apply_and_unknown_settings_keep_the_normal_warning( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + from server.cli.cell_update_config import read_target + + current = _config() + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings:\n future_cell_field: ignored\n tracker_org: next\n") + assert read_target(path, current).tracker_org == "next" + assert "future_cell_field" in caplog.text + path.write_text( + "version: 1\nsettings:\n inference_token_audience: own-audience\n public_base_url: ''\n" + ) + with pytest.raises(SettingsFileError, match="invalid target configuration"): + read_target(path, current) + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("", {"scalar"}), + ("not json", {"scalar"}), + ("[]", {"scalar"}), + ('[{"namespace":"code-only"}]', {"scalar"}), + ( + '[{"namespace":"first","org":"one","repo":"a/b"},' + '{"namespace":"first","org":"duplicate"},' + '{"namespace":"bad.name","org":"skipped"},' + '{"namespace":"second","org":"two"}]', + {"one", "two"}, + ), + ], +) +def test_explicit_code_parser_keeps_process_semantics_without_registry_or_gauge_changes( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: set[str] +) -> None: + # The parser retains its existing defensive fallback for a corrupt runtime value. + config = _config(code_org="scalar", code_repo="a/b", code_namespace="code").model_copy( + update={"code_repos_json": raw} + ) + for key in ("code_org", "code_repo", "code_namespace", "code_repos_json"): + monkeypatch.setattr(settings, key, getattr(config, key)) + monkeypatch.setattr(repos, "_config_errors", {}) + monkeypatch.setattr(repos, "_config_errors_logged", set()) + monkeypatch.setattr(repos, "_config_evaluated", False) + process_result = repos.operator_orgs() + errors = dict(repos._config_errors) + logged = set(repos._config_errors_logged) + + def no_registry() -> object: + raise AssertionError("candidate read a member-managed registry") + + monkeypatch.setattr(repos, "registry_repos", no_registry) + assert repos.operator_orgs_for(config) == process_result == expected + assert repos._config_errors == errors + assert repos._config_errors_logged == logged + + +def test_explicit_fingerprint_is_identical_to_the_process_and_uses_candidate_orgs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = _config(code_org="before", tracker_org="tracker") + for name in Settings.model_fields: + monkeypatch.setattr(settings, name, getattr(current, name)) + monkeypatch.setattr(cell, "_code_orgs", lambda: {"extra"}) + fields = cell.fingerprint_for(current, "deployment", {"extra"}) + assert json.dumps(fields, sort_keys=True) == json.dumps( + cell.fingerprint("deployment"), sort_keys=True + ) + target = _config(code_org="after", tracker_org="tracker") + changed = cell.fingerprint_for(target, "deployment", {"target-repo"}) + assert "after" in changed["configured_orgs"] + assert "target-repo" in changed["configured_orgs"] + assert "before" not in changed["configured_orgs"] + assert cell.fingerprint("deployment") == fields + + +@pytest.mark.parametrize( + "template", + [ + "version: 1\nsettings:\n openai_base_url: {marker}\n", + "version: 1\nsettings:\n tracker_org: !!int {marker}\n", + "version: 1\nsettings:\n tracker_org: !!float {marker}\n", + "version: 1\nsettings:\n tracker_org: !!bool {marker}\n", + "version: 1\nsettings:\n tracker_org: !!timestamp {marker}\n", + "version: {marker}\nsettings: {{}}\n", + "version: 1\nsettings:\n jwt_secret: [\n {marker}\n", + "version: 1\nsettings:\n memory_ask_deep_evidence_max_memories: {marker}\n", + "version: 1\nsettings:\n {marker}: first\n {marker}: second\n", + "version: 1\nsettings:\n tracker_org: 'next'\n{marker}: extra\n", + ], +) +def test_invalid_target_errors_never_echo_supplied_values(tmp_path: Path, template: str) -> None: + from server.cli.cell_update_config import read_target + + marker = "private-overlay-regression-marker" + path = tmp_path / "target.yaml" + path.write_text(template.format(marker=marker)) + with pytest.raises(SettingsFileError) as error: + read_target(path, _config()) + assert marker not in str(error.value) + assert error.value.__suppress_context__ + assert "target" in str(error.value) + + +def test_malformed_target_encoding_is_a_safe_configuration_refusal(tmp_path: Path) -> None: + from server.cli.cell_update_config import read_target + + path = tmp_path / "target.yaml" + path.write_bytes(b"version: 1\nsettings:\n tracker_org: \xff\n") + with pytest.raises(SettingsFileError, match="invalid target settings file"): + read_target(path, _config()) diff --git a/tests/unit/test_cell_update_pair.py b/tests/unit/test_cell_update_pair.py new file mode 100644 index 0000000000..fcb89bdfaa --- /dev/null +++ b/tests/unit/test_cell_update_pair.py @@ -0,0 +1,162 @@ +"""A configuration action reserves only its selected pair and preserves unknown fields.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +import pytest + +from server.core import cell +from server.router import directory, fingerprints + + +def value(issuer: str) -> dict[str, Any]: + return cell.as_value({"bearer_issuers": [issuer], "future": {"nested": [False, 0]}}) + + +@pytest.mark.parametrize( + "action,pending,target,want", + [ + ("config", None, "new", ("old", "new")), + ("config", "new", "new", ("old", "new")), + ("config", None, "old", ("old", None)), + ("finish", "new", "new", ("new", None)), + ("revert", "new", "old", ("old", None)), + ("finish", None, "old", ("old", None)), + ("revert", None, "old", ("old", None)), + ], +) +def test_configuration_pair_selects_exact_reserved_values( + action: str, pending: str | None, target: str, want: tuple[str, str | None] +) -> None: + current, next_fp, chosen = value("old"), value(pending) if pending else None, value(target) + before = deepcopy((current, next_fp, chosen)) + result = fingerprints.configuration_pair(current, next_fp, chosen, action) + assert result == (value(want[0]), value(want[1]) if want[1] else None) + assert (current, next_fp, chosen) == before + + +@pytest.mark.parametrize( + "action,pending,target", + [ + ("config", "new", "third"), + ("config", "new", "old"), + ("finish", "new", "old"), + ("revert", "new", "new"), + ("finish", None, "new"), + ("revert", None, "new"), + ("unsupported", "new", "new"), + ], +) +def test_configuration_pair_refuses_a_different_pending_or_selected_target( + action: str, pending: str | None, target: str +) -> None: + with pytest.raises(ValueError): + fingerprints.configuration_pair( + value("old"), value(pending) if pending else None, value(target), action + ) + + +@pytest.mark.parametrize("broken", [None, {"v": True, "fields": {}}, {"v": 2, "fields": {}}]) +def test_configuration_pair_never_invents_an_unreadable_current(broken: Any) -> None: + with pytest.raises(ValueError, match="fingerprint"): + fingerprints.configuration_pair(broken, None, value("new"), "config") + + +def test_configuration_pair_uses_type_sensitive_unknown_field_equality() -> None: + current, pending, target = value("old"), value("new"), value("new") + target["fields"]["future"]["nested"][0] = 0 + assert pending == target # Python equality cannot implement the reservation CAS. + with pytest.raises(ValueError, match="pending"): + fingerprints.configuration_pair(current, pending, target, "config") + + +@pytest.mark.parametrize("action", ["config", "finish", "revert"]) +def test_configuration_pair_selects_scim_for_the_native_owner_precondition(action: str) -> None: + target = value("new") + target["fields"]["scim"] = True + current = target if action == "revert" else value("old") + assert fingerprints.configuration_pair( + current, target if action == "finish" else None, target, action + ) == ((current, target) if action == "config" else (target, None)) + + +def test_configuration_pair_allows_revert_from_an_existing_scim_candidate() -> None: + pending = value("new") + pending["fields"]["scim"] = True + assert fingerprints.configuration_pair(value("old"), pending, value("old"), "revert") == ( + value("old"), + None, + ) + + +@pytest.mark.asyncio +async def test_configuration_refusals_keep_fixed_reasons_without_relaying_unknown_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + private = "private-untrusted-input" + cases = [ + ( + None, + None, + value(private), + "config", + "configuration update needs readable fingerprints", + ), + ( + value("old"), + None, + value(private), + private, + "configuration action must be config, finish or revert", + ), + ( + value("old"), + value("new"), + value(private), + "config", + "another configuration is pending; finish or revert it first", + ), + ( + value("old"), + value("new"), + value(private), + "finish", + "configuration target differs from the selected reserved fingerprint", + ), + ] + + def no_connection(): + raise AssertionError("configuration validation must finish before directory I/O") + + monkeypatch.setattr(directory, "_connection", no_connection) + for current, pending, target, action, reason in cases: + with pytest.raises(directory.OperatorRefused) as outward: + await directory.update_configuration( + "first", + target, + action, + keys_fp="unused", + refusal=lambda _: "", + expected=directory.Control({"current_fp": current, "next_fp": pending}, None), + ) + assert outward.value.reason == reason + assert private not in outward.value.reason + + failure = ValueError(private) + + def unexpected(*args): + raise failure + + monkeypatch.setattr(fingerprints, "configuration_pair", unexpected) + with pytest.raises(ValueError) as unhandled: + await directory.update_configuration( + "first", + value("new"), + "config", + keys_fp="unused", + refusal=lambda _: "", + expected=directory.Control({"current_fp": value("old"), "next_fp": None}, None), + ) + assert unhandled.value is failure diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py new file mode 100644 index 0000000000..5465767496 --- /dev/null +++ b/tests/unit/test_cell_update_participants.py @@ -0,0 +1,301 @@ +"""A finalization census never shrinks by expiry or accepts an old serving identity.""" + +from __future__ import annotations + +import importlib +from uuid import UUID + +import pytest + + +def participant(n, state="running", proof="", version=1, expired=True): + return { + "participant_id": str(UUID(int=n)), + "state": state, + "host_proof": proof, + "version": version, + "expired": expired, + } + + +def test_capture_keeps_expired_drained_and_uncertified_retired(): + module = importlib.import_module("server.cli.cell_update_participants") + values = [participant(1), participant(2, "drained"), participant(3, "retired")] + assert set(module.capture(values)) == {row["participant_id"] for row in values} + + +def test_capture_omits_only_complete_foundation_retirement_certificate(): + module = importlib.import_module("server.cli.cell_update_participants") + good = "host|boot||namespace|1|3|||cell-retirement-v1" + values = [participant(1, "retired", good), participant(2, "retired", "cell-retirement-v1")] + assert set(module.capture(values)) == {str(UUID(int=2))} + + +def test_current_target_in_initial_census_still_must_turn_over(): + module = importlib.import_module("server.cli.cell_update_participants") + row = participant(1, proof="host|boot||namespace|1|3|||cell-retirement-v1") + assert row["participant_id"] in module.capture([row]) + + +def test_census_conflicting_equal_versions_refuse_instead_of_picking_a_replica(): + module = importlib.import_module("server.cli.cell_update_participants") + with pytest.raises(module.cell_control.Refused, match="conflicting"): + module.latest([participant(1), participant(1, "retired")]) + + +@pytest.mark.parametrize("version", [True, "2", 0, -1, None]) +def test_census_refuses_unreadable_versions(version): + from server.cli import cell_control + from server.cli import cell_update_participants as module + + with pytest.raises(cell_control.Refused, match="unreadable"): + module.latest([participant(1, version=version)]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "change", ["missing", "drained", "expired", "uncertified", "old-version", "other-proof"] +) +async def test_retirement_requires_permanent_exact_identity_and_newer_version(monkeypatch, change): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + proof = "host|boot||namespace|1|3|||cell-retirement-v1" + initial = participant(1, proof=proof, version=2) + current = participant(1, "retired", proof=proof, version=3) + if change == "missing": + current = None + elif change in ("drained", "expired"): + current["state"] = "drained" if change == "drained" else "running" + elif change == "uncertified": + current["host_proof"] = "" + elif change == "old-version": + current["version"] = 2 + else: + current["host_proof"] = proof.replace("|1|3|", "|2|3|") + monkeypatch.setattr(module, "proves_stopped", lambda _: False) + ch = type("CH", (), {"query_one": AsyncMock(return_value={"running": 0})})() + assert not await module.stopped(ch, initial, current) + ch.query_one.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_exact_retirement_certificate_is_accepted_without_age_or_query_heuristic(): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + proof = "host|boot||namespace|1|3|||cell-retirement-v1" + ch = type("CH", (), {"query_one": AsyncMock(side_effect=AssertionError("no heuristic"))})() + assert await module.stopped( + ch, participant(1, proof=proof), participant(1, "retired", proof, 2) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "running", [None, {}, {"running": "0"}, {"running": False}, {"running": 1}, {"running": 0}] +) +async def test_physical_death_also_requires_known_zero_strict_query_census(monkeypatch, running): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + monkeypatch.setattr(module, "proves_stopped", lambda _: True) + ch = type("CH", (), {"query_one": AsyncMock(return_value=running)})() + assert await module.stopped(ch, participant(1), None) is ( + running == {"running": 0} and type(running["running"]) is int + ) + assert "skip_unavailable_shards=0" in ch.query_one.call_args.args[0] + + +@pytest.mark.asyncio +async def test_physical_death_query_failure_never_falls_back_to_local(monkeypatch): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + monkeypatch.setattr(module, "proves_stopped", lambda _: True) + ch = type("CH", (), {"query_one": AsyncMock(side_effect=OSError("replica unavailable"))})() + with pytest.raises(OSError, match="replica unavailable"): + await module.stopped(ch, participant(1), None) + assert ch.query_one.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "change", + [ + "valid", + "worker", + "captured", + "retired", + "unmarked", + "wrong-cell", + "old-fingerprint", + "no-instance", + "no-participant", + "not-router", + ], +) +async def test_replacement_joins_serving_instance_and_guarded_new_participant(monkeypatch, change): + import json + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + from server.core import cell + + identity = str(UUID(int=2)) + target = cell.as_value({"deployment_id": "dep"}) + proof = "host|boot||namespace|1|3|||cell-retirement-v1" + rows = {identity: participant(2, proof=proof)} + captured = {str(UUID(int=1))} + live = [{"instance_id": "replacement", "serving": change != "worker"}] + value = { + "participant_id": identity, + "fields": target["fields"], + "serves": True, + "marked": True, + "router": True, + } + if change == "captured": + captured.add(identity) + elif change == "retired": + rows[identity]["state"] = "retired" + elif change == "unmarked": + value["marked"] = False + elif change == "not-router": + value["router"] = False + elif change == "old-fingerprint": + value["fields"] = {"deployment_id": "old"} + elif change == "no-instance": + live = [] + elif change == "no-participant": + rows = {} + beat = { + "instance_id": "replacement", + "cell": "other" if change == "wrong-cell" else "first", + "tenancy_mode": "enforced", + "fingerprint": json.dumps(value), + } + monkeypatch.setattr(module.instances, "read_live", AsyncMock(return_value=live)) + ch = SimpleNamespace(query_all=AsyncMock(return_value=[beat])) + run = SimpleNamespace(args=SimpleNamespace(cell="first"), row={"is_first": True}) + assert await module.replacement(ch, run, rows, captured, target) is (change == "valid") + assert "use_query_cache = '0'" in ch.query_all.call_args.args[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "engine", ["ReplacingMergeTree", "ReplicatedReplacingMergeTree", "SharedReplacingMergeTree"] +) +async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + from server.cli.cell_ch import Topology + + events = [] + topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) + monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) + monkeypatch.setattr( + module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + ) + monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + + async def per_host(ch, top, names, sql, params): + if "source:tables" in sql: + return {name: [{"engine": engine}] for name in names} + return { + name: [ + { + "database": "loom", + "table": "_provider_participants", + "zookeeper_path": "/same", + "replica_name": name, + "replica_is_active": {"a": 1, "b": 1}, + } + ] + for name in names + } + + async def sync(*args): + events.append("sync") + + async def query(sql, params): + events.append("read") + assert "clusterAllReplicas" in sql + assert "skip_unavailable_shards=0" in sql + assert "result_overflow_mode = 'throw'" in sql + assert "use_query_cache = '0'" in sql + assert f"select_sequential_consistency={int(engine.startswith('Shared'))}" in sql + return [participant(1), participant(2), participant(1, "drained", version=2)] + + monkeypatch.setattr(module, "per_host", per_host) + monkeypatch.setattr(module.cell_scan, "_system", sync) + result = await module.census(SimpleNamespace(query_all=query)) + assert len(result) == 2 and result[str(UUID(int=1))]["state"] == "drained" + assert events == (["sync", "read"] if engine.startswith("Replicated") else ["read"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "problem", + ["unlisted", "shards", "missing-table", "unreachable", "extra-replica", "sync-failure"], +) +async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkeypatch, problem): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + from server.cli.cell_ch import Topology + + topology = Topology( + "replicas", + 0 if problem == "unlisted" else 2 if problem == "shards" else 1, + () if problem == "unlisted" else (("a", 9000), ("b", 9000)), + ) + monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) + monkeypatch.setattr( + module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + ) + monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + + async def per_host(*args): + if problem == "unreachable": + raise OSError("replica unavailable") + if "source:tables" in args[3]: + return { + "a": [{"engine": "ReplicatedReplacingMergeTree"}], + "b": [] + if problem == "missing-table" + else [{"engine": "ReplicatedReplacingMergeTree"}], + } + return { + name: [ + { + "database": "loom", + "table": "_provider_participants", + "zookeeper_path": "/same", + "replica_name": name, + "replica_is_active": { + "a": 1, + "b": 1, + **({"unknown": 1} if problem == "extra-replica" else {}), + }, + } + ] + for name in ("a", "b") + } + + monkeypatch.setattr(module, "per_host", per_host) + monkeypatch.setattr( + module.cell_scan, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) + ) + query = AsyncMock(side_effect=AssertionError("must not read partial census")) + with pytest.raises((module.cell_control.Refused, OSError)): + await module.census(SimpleNamespace(query_all=query)) + query.assert_not_awaited() diff --git a/tests/unit/test_cell_update_probe.py b/tests/unit/test_cell_update_probe.py new file mode 100644 index 0000000000..fdcdb7d73a --- /dev/null +++ b/tests/unit/test_cell_update_probe.py @@ -0,0 +1,74 @@ +"""A configuration or address proof cannot leave an unbounded HTTP waiter.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control, cell_update_probe +from tests.unit.test_cell_update_run import row + + +@pytest.mark.asyncio +async def test_update_proof_has_whole_reply_deadline_and_exact_nonce_cleanup(monkeypatch): + class Direct: + conn = object() + + async def call(self, work, *, total_s): + return await work() + + nonce = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_update_probe.cell_prepare, "_new_nonces", lambda: nonce) + monkeypatch.setattr(cell_update_probe.cell_prepare, "_write_nonces", AsyncMock()) + cleanup = AsyncMock() + monkeypatch.setattr(cell_update_probe.cell_prepare, "_delete_nonces", cleanup) + monkeypatch.setattr(cell_update_probe, "ANSWER_S", 0.02, raising=False) + cancelled = asyncio.Event() + + async def stalled(*args, **kwargs): + try: + await asyncio.Future() + finally: + cancelled.set() + + monkeypatch.setattr(cell_update_probe.cell_prepare, "_register", stalled) + run = SimpleNamespace( + owner=Direct(), + args=SimpleNamespace(wait=0), + ch=object(), + router=None, + transport=None, + row=row(), + ) + task = asyncio.create_task(cell_update_probe.post(run, "/configuration", {})) + done, _ = await asyncio.wait({task}, timeout=0.15) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + pytest.fail("update proof exceeded its whole reply deadline") + with pytest.raises(TimeoutError): + await task + assert cancelled.is_set() + cleanup.assert_awaited_once_with(run.ch, nonce) + + +@pytest.mark.asyncio +async def test_router_without_update_route_names_the_required_upgrade(): + import httpx + + from server.cli import cell_prepare + + with pytest.raises(cell_prepare.Refused, match="not upgraded for cell update"): + await cell_prepare._register( + SimpleNamespace(cell="first", wait=0), + httpx.URL("http://router"), + {}, + httpx.MockTransport(lambda _: httpx.Response(404, text="Not Found")), + path="/configuration", + command="update", + ) diff --git a/tests/unit/test_cell_update_run.py b/tests/unit/test_cell_update_run.py new file mode 100644 index 0000000000..28718be2ee --- /dev/null +++ b/tests/unit/test_cell_update_run.py @@ -0,0 +1,231 @@ +"""Cross-store update ordering and rerun choices use only recorded fingerprints.""" + +from __future__ import annotations + +from argparse import Namespace +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import httpx +import pytest + +from server.cli import cell_control, cell_update_participants, cell_update_pg, cell_update_run +from server.core import cell + + +def value(issuer="old", *, scim=False, orgs=()): + return cell.as_value( + { + "deployment_id": "dep-first", + "public_host": "first.example.test", + "oauth_issuer_host": "first.example.test", + "bearer_issuers": [issuer], + "configured_orgs": list(orgs), + "scim": scim, + } + ) + + +def row(current=None, pending=None): + return { + "cell": "first", + "is_first": True, + "host": "first.example.test", + "deployment_id": "dep-first", + "bucket": "", + "bucket_prefix": "", + "probe_url": "http://probe", + "upstream_url": "http://main", + "current_fp": current, + "next_fp": pending, + "mutation_generation": str(UUID(int=1)), + "state": "serving", + "mode": "enforced", + } + + +@pytest.fixture +def coordinator(monkeypatch): + events = [] + run = cell_update_run.Run( + None, + Namespace(cell="first", config=None, finish=False, revert=False, probe_url=None, wait=0), + httpx.URL("http://router"), + None, + ) + run.owner = SimpleNamespace(conn=object()) + run.row = row(value(), value("new", orgs=("held",))) + run.state = cell_update_pg.State("dep-first", True, "first", "enforced", value(), None) + run.target = None + + async def held(current, pending): + events.append("placement-proof") + return ["held"] + + async def write(owner, expected, registered, current, pending, placed): + events.append("local-write") + assert registered is run.row + assert placed == ["held"] or registered["current_fp"] is None + run.state = replace(expected, current=current, pending=pending) + return run.state + + async def durable(): + events.append("durable") + return run.state + + async def publish(target, action): + events.append("directory-" + action) + current, pending = cell_update_run.fingerprints.configuration_pair( + run.row["current_fp"], run.row["next_fp"], target, action + ) + return run.row | {"current_fp": current, "next_fp": pending} + + async def settle(selected, target): + events.append("turnover") + assert selected is run + assert run.state.current == target and run.state.pending is None + assert run.row["next_fp"] is not None + + async def call(path): + events.append("directory-read") + assert path == "" + return run.row + + monkeypatch.setattr(run, "held_orgs", held) + monkeypatch.setattr(run, "durable", durable) + monkeypatch.setattr(run, "publish", publish) + monkeypatch.setattr(run, "call", call) + monkeypatch.setattr(cell_update_pg, "write", write) + monkeypatch.setattr(cell_update_participants, "settle", settle) + return run, events + + +@pytest.mark.asyncio +async def test_config_reserves_directory_before_any_local_acceptance(coordinator): + run, events = coordinator + run.args.config, run.target = Path("target"), value("new", orgs=("held",)) + await run.configuration() + assert events == ["directory-config", "placement-proof", "local-write", "durable"] + assert run.state.pending == run.target + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["finish", "revert"]) +@pytest.mark.parametrize("already_narrowed", [False, True]) +async def test_finalization_repairs_then_cuts_off_then_turns_over_then_releases( + coordinator, action, already_narrowed +): + run, events = coordinator + setattr(run.args, action, True) + target = run.row["next_fp"] if action == "finish" else run.row["current_fp"] + if already_narrowed: + run.state = replace(run.state, current=target, pending=None) + await run.configuration() + assert events == [ + "placement-proof", + "local-write", + "durable", + "turnover", + "durable", + "directory-read", + "directory-" + action, + ] + assert run.state.current == target and run.state.pending is None + assert run.row["current_fp"] == target and run.row["next_fp"] is None + + +@pytest.mark.asyncio +async def test_cutoff_timeout_keeps_directory_dual_without_compensating_local_rewiden( + coordinator, monkeypatch +): + run, events = coordinator + run.args.finish = True + monkeypatch.setattr( + cell_update_participants, "settle", AsyncMock(side_effect=TimeoutError("busy")) + ) + before = run.row.copy() + with pytest.raises(TimeoutError, match="busy"): + await run.configuration() + assert run.row == before + assert run.state.current == before["next_fp"] and run.state.pending is None + assert events == ["placement-proof", "local-write", "durable"] + + +@pytest.mark.asyncio +async def test_prebegin_never_reads_or_places_configured_orgs(coordinator, monkeypatch): + run, events = coordinator + run.row = row() + run.args.config, run.target = Path("target"), value("new", orgs=("unplaced",)) + forbidden = AsyncMock(side_effect=AssertionError("directory reservation before begin")) + monkeypatch.setattr(run, "held_orgs", forbidden) + monkeypatch.setattr(run, "publish", forbidden) + await run.configuration() + assert events == ["local-write", "durable"] + assert run.state.pending == run.target + + +@pytest.mark.asyncio +async def test_lost_generation_after_barrier_never_releases_reservation(coordinator, monkeypatch): + run, events = coordinator + run.args.finish = True + monkeypatch.setattr( + run, "call", AsyncMock(return_value=run.row | {"mutation_generation": str(UUID(int=2))}) + ) + with pytest.raises(cell_control.Refused, match="ownership or reservation changed"): + await run.configuration() + assert "directory-finish" not in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["config", "finish", "revert"]) +async def test_singleton_noop_proves_local_durability_without_turnover(coordinator, action): + run, events = coordinator + run.row["next_fp"] = None + run.state = replace(run.state, current=run.row["current_fp"], pending=None) + if action == "config": + run.args.config, run.target = Path("target"), run.row["current_fp"] + else: + setattr(run.args, action, True) + await run.configuration() + assert events.count("durable") == 1 + assert "turnover" not in events + assert "directory-" + action in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["config", "finish"]) +async def test_scim_on_refuses_before_generation_claim_or_acceptance( + coordinator, monkeypatch, action +): + run, _ = coordinator + unsafe = value("new", scim=True) + run.row["next_fp"] = unsafe + if action == "config": + run.args.config = Path("target") + monkeypatch.setattr(cell_update_run.cell_update_config, "read_target", lambda *_: object()) + monkeypatch.setattr( + cell_update_run.cell_update_config, "target_fingerprint", lambda *_: unsafe + ) + else: + run.args.finish = True + claim = AsyncMock(side_effect=AssertionError("must not claim")) + monkeypatch.setattr(cell_control, "claim", claim) + with pytest.raises(cell_control.Refused, match="census client differs"): + await run.start(run.owner) + claim.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_revert_repairs_tenancy_for_pending_placements_before_dropping_fingerprint( + coordinator, monkeypatch +): + run, _ = coordinator + run.args.revert = True + held = AsyncMock(return_value=["held"]) + monkeypatch.setattr(run, "held_orgs", held) + before = run.row.copy() + await run.configuration() + held.assert_awaited_once_with(before["current_fp"], before["next_fp"]) diff --git a/tests/unit/test_cell_update_scim_precondition.py b/tests/unit/test_cell_update_scim_precondition.py new file mode 100644 index 0000000000..e4376ac90f --- /dev/null +++ b/tests/unit/test_cell_update_scim_precondition.py @@ -0,0 +1,44 @@ +"""A valid SCIM target reaches the update owner's provisioning precondition.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from server.cli.cell_update_config import target_fingerprint +from server.router import fingerprints +from tests.unit.test_cell_update_pair import value +from tests.unit.test_cell_update_target import REGISTERED, _config + + +@pytest.mark.parametrize("scim_on", [True, False], ids=["scim-on", "scim-off"]) +def test_target_checks_the_real_cell_token_policy_without_a_blanket_refusal(scim_on): + config = _config( + scim_token="lsc_first_" + "a" * 64 if scim_on else "", + oauth_login_provider="oidc", + auth_required=True, + ) + registered = {**REGISTERED, "is_first": True} + before = deepcopy(registered) + try: + result = target_fingerprint(config, "deployment", registered) + except ValueError as error: + result = error + assert not isinstance(result, ValueError), str(result) + assert result["fields"]["scim"] is scim_on + assert registered == before + + +@pytest.mark.parametrize("scim_on", [True, False], ids=["scim-on", "scim-off"]) +def test_pair_selects_the_reserved_target_for_the_real_precondition_owner(scim_on): + current, target = value("old"), value("new") + target["fields"]["scim"] = scim_on + before = deepcopy((current, target)) + try: + result = fingerprints.configuration_pair(current, None, target, "config") + except ValueError as error: + result = error + assert not isinstance(result, ValueError), str(result) + assert result == (current, target) + assert (current, target) == before diff --git a/tests/unit/test_cell_update_scim_runtime.py b/tests/unit/test_cell_update_scim_runtime.py new file mode 100644 index 0000000000..2cbb67094d --- /dev/null +++ b/tests/unit/test_cell_update_scim_runtime.py @@ -0,0 +1,244 @@ +"""Update checks the selected SCIM policy before claiming and releasing a reservation.""" + +from __future__ import annotations + +from argparse import Namespace +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import httpx +import pytest + +from server.auth import scim +from server.cli import cell_control, cell_update_participants, cell_update_probe, cell_update_run +from server.cli.cell_ch import Refused +from server.cli.cell_update_pg import State +from server.core import cell, pg_profile +from server.settings import settings +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run + +EMAIL = "person@example.test" + + +@pytest.fixture +def update_run(provisioned_run, monkeypatch, tmp_path): + attached, people, native = provisioned_run + native["mode"] = "enforced" + monkeypatch.setattr(cell, "mode", lambda: "enforced") + monkeypatch.setattr(cell, "deployment_id", lambda: "dep-1") + for name, value in { + "public_base_url": "https://first.example.test", + "oauth_issuer": "https://first.example.test", + "document_s3_prefix": "originals", + "code_repos_json": "[]", + "code_org": "", + "tracker_org": "", + "oidc_jit_org": "", + "self_signup": "off", + "oidc_client_id": "", + "grant_cache_ttl_seconds": 240, + }.items(): + monkeypatch.setattr(settings, name, value) + monkeypatch.setattr(pg_profile, "standby_now", AsyncMock(return_value="synchronous")) + fields = cell.fingerprint_for(settings, "dep-1", set()) + current = cell.as_value({**fields, "scim": False}) + target = cell.as_value({**fields, "oidc_client_id": "pending"}) + config = tmp_path / "target.yaml" + config.write_text("version: 1\nsettings:\n oidc_client_id: pending\n") + args = Namespace( + cell="first", + config=None, + finish=False, + revert=False, + probe_url=None, + wait=0, + revoke_unprovisioned=False, + ) + run = cell_update_run.Run(people, args, httpx.URL("http://router"), None) + run.owner = owner(attached.lock_conn) + run.target = None + run.row = { + **attached.row, + "host": "first.example.test", + "probe_url": "http://probe", + "upstream_url": "http://main", + "mutation_generation": str(UUID(int=1)), + "state": "serving", + } + events = [] + control = SimpleNamespace(config=config, reads=[], after_turnover=lambda: None) + + def pair(active, pending): + attached.own_fp = (active, pending) + run.row.update(current_fp=active, next_fp=pending) + run.state = State("dep-1", True, "first", "enforced", active, pending) + + control.pair = pair + pair(current, target) + answer = run.owner.conn._answer + + def read(sql, params): + control.reads.append(sql) + return answer(sql, params) + + execute = run.owner.conn.execute + + async def write(sql, *params): + result = await execute(sql, *params) + if sql.startswith("UPDATE loom.cell_self SET current_fp="): + events.append("local-write") + attached.own_fp = params + return "UPDATE 1" + return result + + async def call(path, body=None): + events.append(path or "directory-read") + if path == "": + return dict(run.row) + if path == "/claim": + assert body == cell_control.preconditions(run.row, claim=True) + return {**run.row, "mutation_generation": str(UUID(int=2))} + assert path == "/configuration/placements" + assert body["orgs"] == [] + return {"orgs": []} + + async def post(selected, path, body): + assert selected is run and path == "/configuration" + action = body["action"] + events.append("publish-" + action) + active, pending = cell_update_run.fingerprints.configuration_pair( + run.row["current_fp"], run.row["next_fp"], body["target_fp"], action + ) + return {**run.row, "current_fp": active, "next_fp": pending} + + async def settle(selected, target): + assert selected is run + assert (run.state.current, run.state.pending) == (target, None) + events.append("turnover") + control.after_turnover() + + monkeypatch.setattr(run.owner.conn, "_answer", read) + monkeypatch.setattr(run.owner.conn, "execute", write) + monkeypatch.setattr(run, "call", call) + monkeypatch.setattr(cell_update_probe, "post", post) + monkeypatch.setattr(cell_update_participants, "settle", settle) + return run, people, events, control + + +async def test_config_provisioned_target_passes_actual_census_on_both_local_pairs(update_run): + run, people, events, control = update_run + current, target = run.state.current, run.state.pending + control.pair(current, None) + run.args.config = control.config + people.accounts = [("o", EMAIL, EMAIL, True)] + people.provision(EMAIL) + + result = await run.start(run.owner) + + assert result["action"] == "config" and result["next"] == cell.digest(target["fields"]) + assert run.target == target + assert (run.state.current, run.state.pending) == (current, target) + assert (run.row["current_fp"], run.row["next_fp"]) == (current, target) + assert len(people.census_reads) == 6 + assert events.index("/claim") < events.index("publish-config") < events.index("local-write") + assert "turnover" not in events and people.writes == [] + + +@pytest.mark.parametrize("action", ["config", "finish"]) +async def test_unverified_contact_refuses_before_generation_claim_or_acceptance(update_run, action): + run, people, events, control = update_run + if action == "config": + control.pair(run.state.current, None) + run.args.config = control.config + else: + run.args.finish = True + people.accounts = [("o", "handle", EMAIL, False)] + people.provision(EMAIL) + before = (run.state.current, run.state.pending, run.row["mutation_generation"]) + + with pytest.raises(Refused, match="unprovisioned accounts subject 'handle'"): + await run.start(run.owner) + + assert (run.state.current, run.state.pending, run.row["mutation_generation"]) == before + assert events == ["directory-read"] + assert len(people.census_reads) == 1 + assert not any( + sql.startswith("UPDATE ") or sql.startswith("INSERT ") for sql in run.owner.conn.executed + ) + assert people.writes == [] + + +async def test_finish_provisioned_target_rechecks_and_releases_after_turnover(update_run): + run, people, events, _control = update_run + run.args.finish = True + target = run.state.pending + people.accounts = [("o", EMAIL, EMAIL, True)] + people.provision(EMAIL) + + result = await run.start(run.owner) + + assert result["action"] == "finish" and result["next"] is None + assert (run.state.current, run.state.pending) == (target, None) + assert (run.row["current_fp"], run.row["next_fp"]) == (target, None) + assert len(people.census_reads) == 9 + assert events.index("local-write") < events.index("turnover") < events.index("publish-finish") + assert people.writes == [] + + +async def test_finish_fresh_census_refuses_remote_deprovisioning_before_release(update_run): + run, people, events, control = update_run + run.args.finish = True + current, target = run.state.current, run.state.pending + people.accounts = [("o", EMAIL, EMAIL, True)] + people.provision(EMAIL) + assert not await scim.refuses_subject(EMAIL, org="o") + control.after_turnover = people.users.clear + + with pytest.raises(Refused, match="unprovisioned accounts subject"): + await run.start(run.owner) + + assert not await scim.refuses_subject(EMAIL, org="o"), "the ordinary permit is still warm" + assert (run.state.current, run.state.pending) == (target, None) + assert (run.row["current_fp"], run.row["next_fp"]) == (current, target) + assert len(people.census_reads) == 7 + assert "turnover" in events and "publish-finish" not in events + assert people.writes == [] + + +async def test_revert_selects_scim_off_current_without_reading_provisioning(update_run): + run, people, events, _control = update_run + run.args.revert = True + current = run.state.current + people.unavailable = True + + result = await run.start(run.owner) + + assert result["action"] == "revert" and result["next"] is None + assert (run.state.current, run.state.pending) == (current, None) + assert (run.row["current_fp"], run.row["next_fp"]) == (current, None) + assert events.index("turnover") < events.index("publish-revert") + assert people.census_reads == people.reads == people.writes == [] + + +async def test_scim_off_precondition_needs_no_store_io(update_run): + run, people, events, control = update_run + people.unavailable = True + + await run.scim_precondition(run.state.current) + + assert control.reads == run.owner.conn.executed == events == [] + assert people.census_reads == people.reads == people.writes == [] + + +async def test_foreign_target_refuses_before_identity_reads_or_claim(update_run): + run, people, events, _control = update_run + target = cell.as_value({**run.state.pending["fields"], "deployment_id": "foreign"}) + + with pytest.raises(Refused, match="intended configuration names another deployment"): + await run.scim_precondition(target) + + assert events == [] + assert people.census_reads == [] + assert people.writes == [] diff --git a/tests/unit/test_cell_update_target.py b/tests/unit/test_cell_update_target.py new file mode 100644 index 0000000000..572fe43996 --- /dev/null +++ b/tests/unit/test_cell_update_target.py @@ -0,0 +1,126 @@ +"""An update validates its fingerprint before reserving or accepting it.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from typing import Any + +import pytest + +from server.core import cell +from server.settings import Settings, settings + +REGISTERED = { + "cell": "first", + "host": "loom.example", + "deployment_id": "deployment", + "bucket": "docs", + "bucket_prefix": "originals", +} + + +def _config(**overrides: Any) -> Settings: + values = settings.model_dump() | { + "public_base_url": "https://loom.example", + "oauth_issuer": "https://loom.example", + "document_s3_bucket": "docs", + "document_s3_prefix": "originals", + "code_repos_json": "[]", + "code_org": "", + "tracker_org": "", + "oidc_jit_org": "", + "self_signup": "off", + "scim_token": "", + } + return Settings(_env_file=None, **(values | overrides)) + + +def test_target_uses_explicit_settings_and_the_existing_code_reader() -> None: + from server.cli.cell_update_config import target_fingerprint + + config = _config(code_org="configured", tracker_org="tracker") + before_config, before_registration = config.model_dump(), deepcopy(REGISTERED) + value = target_fingerprint(config, "deployment", REGISTERED) + assert value == cell.as_value(cell.fingerprint_for(config, "deployment", {"configured"})) + assert value["fields"]["configured_orgs"] == ["configured", "tracker"] + assert config.model_dump() == before_config + assert REGISTERED == before_registration + + +@pytest.mark.parametrize( + "change,field", + [ + ({"document_s3_bucket": "elsewhere"}, "bucket"), + ({"document_s3_prefix": "elsewhere"}, "bucket_prefix"), + ({"public_base_url": "https://elsewhere.example"}, "public_host"), + ({"oauth_issuer": "https://elsewhere.example"}, "oauth_issuer_host"), + ], +) +def test_target_refuses_changed_registered_identity_without_echoing_values( + change: dict[str, str], field: str +) -> None: + from server.cli.cell_update_config import target_fingerprint + + with pytest.raises(ValueError, match=field) as error: + target_fingerprint(_config(**change), "deployment", REGISTERED) + assert "elsewhere" not in str(error.value) + + +def test_target_refuses_different_deployment_and_bucketless_adoption() -> None: + from server.cli.cell_update_config import target_fingerprint + + with pytest.raises(ValueError, match="deployment_id"): + target_fingerprint(_config(), "different", REGISTERED) + with pytest.raises(ValueError, match="bucket"): + target_fingerprint( + _config(), "deployment", REGISTERED | {"bucket": "", "bucket_prefix": ""} + ) + + +def test_target_refuses_invalid_scim_credentials_before_any_state_work() -> None: + from server.cli.cell_update_config import target_fingerprint + + with pytest.raises(ValueError, match="SCIM") as error: + target_fingerprint( + _config(scim_token="private-scim-token"), + "deployment", + REGISTERED | {"is_first": True}, + ) + assert "private-scim-token" not in str(error.value) + + +@pytest.mark.parametrize("org", ["-", "*", "x" * (cell.ORG_MAX_BYTES + 1)]) +def test_target_refuses_reserved_or_oversized_configured_orgs(org: str) -> None: + from server.cli.cell_update_config import target_fingerprint + + # A fully validated Settings model may still name a non-tenant configured org. + with pytest.raises(ValueError, match="fingerprint"): + target_fingerprint(_config(tracker_org=org), "deployment", REGISTERED) + + +def test_target_must_fit_a_live_placement() -> None: + from server.cli.cell_update_config import target_fingerprint + + config = _config( + code_repos_json=json.dumps( + [ + {"namespace": f"repo{n}", "org": str(n) + "x" * 1800, "repo": "owner/project"} + for n in range(35) + ] + ) + ) + with pytest.raises(ValueError, match="placement body"): + target_fingerprint(config, "deployment", REGISTERED) + + +def test_registration_check_applies_defaults_and_preserves_newer_fields() -> None: + from server.router.fingerprints import registration_refusal + + value = cell.as_value({"future": {"nested": [False, 0]}}) + before = deepcopy(value) + empty = REGISTERED | {"host": "", "deployment_id": "", "bucket": "", "bucket_prefix": ""} + assert registration_refusal(value, empty) == "" + assert value == before + assert "deployment_id" in registration_refusal(value, REGISTERED) + assert "unreadable" in registration_refusal({"v": True, "fields": {}}, empty) diff --git a/tests/unit/test_code_repo_entry_clone_fields.py b/tests/unit/test_code_repo_entry_clone_fields.py index b03e462065..1366d5d555 100644 --- a/tests/unit/test_code_repo_entry_clone_fields.py +++ b/tests/unit/test_code_repo_entry_clone_fields.py @@ -25,7 +25,6 @@ def _configure(monkeypatch: pytest.MonkeyPatch, entries: list[dict]) -> None: monkeypatch.setattr(settings, "code_git_dir", "var/code_repos/clickhouse.git") monkeypatch.setattr(settings, "code_git_remote", "https://github.com/ClickHouse/ClickHouse.git") monkeypatch.setattr(settings, "code_repos_json", json.dumps(entries)) - monkeypatch.setattr(repos, "_publish_config_errors", lambda errors: captured.update(errors)) captured: dict[str, str] = {} @@ -46,7 +45,7 @@ def test_release_branch_of_the_scalar_repo_still_inherits_its_clone( {"namespace": "code-clickhouse-26-1", "git_branch": "26.1"}, ], ) - by_ns = {c.namespace: c for c in repos._configured_from_env()} + by_ns = {c.namespace: c for c in repos._configured_from_env(settings, captured)} rel = by_ns["code-clickhouse-26-1"] assert rel.git_dir == "var/code_repos/clickhouse.git" assert rel.git_remote == "https://github.com/ClickHouse/ClickHouse.git" @@ -65,7 +64,7 @@ def test_a_different_repo_derives_its_remote_from_repo(monkeypatch: pytest.Monke } ], ) - (cfg,) = repos._configured_from_env() + (cfg,) = repos._configured_from_env(settings, captured) assert cfg.git_remote == "https://github.com/langfuse/langfuse.git" assert cfg.git_dir == "var/code_repos/langfuse.git" assert not captured @@ -83,7 +82,7 @@ def test_an_explicit_remote_is_kept(monkeypatch: pytest.MonkeyPatch) -> None: } ], ) - (cfg,) = repos._configured_from_env() + (cfg,) = repos._configured_from_env(settings, captured) assert cfg.git_remote == "git@github.com:acme/x.git" @@ -97,7 +96,7 @@ def test_a_different_repo_without_git_dir_gets_a_clone_of_its_own( {"namespace": "code-langfuse", "repo": "langfuse/langfuse"}, ], ) - by_ns = {c.namespace: c for c in repos._configured_from_env()} + by_ns = {c.namespace: c for c in repos._configured_from_env(settings, captured)} lf = by_ns["code-langfuse"] assert lf.git_dir == "var/code_repos/langfuse.git" assert lf.git_dir != by_ns["code-clickhouse"].git_dir diff --git a/tests/unit/test_code_scip_extra_producers.py b/tests/unit/test_code_scip_extra_producers.py index 4a157acc96..dc3edaa2a8 100644 --- a/tests/unit/test_code_scip_extra_producers.py +++ b/tests/unit/test_code_scip_extra_producers.py @@ -68,7 +68,7 @@ def test_the_entry_json_carries_the_extra_producers_and_the_projects(monkeypatch "scip_ts_projects": "frontend, sdk/typescript", } monkeypatch.setattr(settings, "code_repos_json", json.dumps([entry])) - (cfg,) = repos._configured_from_env() + (cfg,) = repos._configured_from_env(settings, {}) assert cfg.scip_extra_producers == ("typescript",) assert cfg.scip_ts_projects == ("frontend", "sdk/typescript") diff --git a/tests/unit/test_directory_pool_init.py b/tests/unit/test_directory_pool_init.py index ea8bb3fd37..978a3fae01 100644 --- a/tests/unit/test_directory_pool_init.py +++ b/tests/unit/test_directory_pool_init.py @@ -15,7 +15,9 @@ ) @pytest.mark.parametrize("position", [0, 1]) async def test_every_failed_initialization_quarantines_its_exact_raw_session(error, position): - conn = Mock(set_type_codec=AsyncMock(side_effect=[None] * position + [error])) + conn = Mock( + set_type_codec=AsyncMock(side_effect=[None] * position + [error]), close=AsyncMock() + ) with pytest.raises(type(error)) as caught: await directory._init(conn) assert caught.value is error @@ -61,7 +63,9 @@ def error(which): original = error(kind) conn = Mock( - set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=error(secondary)) + set_type_codec=AsyncMock(side_effect=original), + close=AsyncMock(), + terminate=Mock(side_effect=error(secondary)), ) if isinstance(original, (SystemExit, KeyboardInterrupt)): with pytest.raises(directory._PoolInitFailure) as caught: @@ -83,7 +87,9 @@ def add_note(self, note): original = BrokenNote(17) conn = Mock( - set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=RuntimeError) + set_type_codec=AsyncMock(side_effect=original), + close=AsyncMock(), + terminate=Mock(side_effect=RuntimeError), ) with pytest.raises(directory._PoolInitFailure) as caught: await directory._init(conn) @@ -158,3 +164,176 @@ async def acquire(): "driver creation cleanup" if boundary == "creation" else "driver acquisition cleanup" ] assert directory._lookups._value == slots + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_failed_codec_does_not_return_before_raw_disconnect(kind): + original = { + "ordinary": ValueError("codec failed"), + "cancel": asyncio.CancelledError("codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("codec interrupted"), + }[kind] + closing, disconnected = asyncio.Event(), asyncio.Event() + terminated = False + + class Raw: + async def set_type_codec(self, *args, **kwargs): + raise original + + async def close(self, *, timeout): # noqa: ASYNC109 — fake driver close API + assert timeout == directory.STATEMENT_TIMEOUT_S + closing.set() + await disconnected.wait() + + def terminate(self): + nonlocal terminated + terminated = True + + async def acquire(): + try: + await directory._init(Raw()) + except directory._PoolInitFailure as failed: + return failed.error + except BaseException as error: # noqa: BLE001 — capture native originals inside the task. + return error + pytest.fail("failed initialization returned success") + + task = asyncio.create_task(acquire()) + started = asyncio.create_task(closing.wait()) + try: + async with asyncio.timeout(1): + await asyncio.wait((task, started), return_when=asyncio.FIRST_COMPLETED) + assert not task.done(), "initializer returned without acknowledging raw disconnect" + assert not terminated + disconnected.set() + assert await asyncio.wait_for(task, 1) is original + assert terminated + finally: + disconnected.set() + for pending in (task, started): + if not pending.done(): + pending.cancel() + await asyncio.gather(task, started, return_exceptions=True) + + +@pytest.mark.parametrize("secondary", ["ordinary", "timeout", "cancel", "exit", "keyboard"]) +async def test_failed_codec_close_failure_keeps_original_and_reports_unknown(secondary): + original = ValueError("original codec failure") + cleanup = { + "ordinary": RuntimeError("close failed"), + "timeout": TimeoutError("close deadline expired"), + "cancel": asyncio.CancelledError("close cancelled"), + "exit": SystemExit(21), + "keyboard": KeyboardInterrupt("close interrupted"), + }[secondary] + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), close=AsyncMock(side_effect=cleanup) + ) + with pytest.raises(ValueError) as caught: + await directory._init(conn) + assert caught.value is original + conn.close.assert_awaited_once_with(timeout=directory.STATEMENT_TIMEOUT_S) + conn.terminate.assert_called_once_with() + assert original.__notes__ == ["directory initialization session closure is unknown"] + + +async def test_expired_acquisition_does_not_start_a_new_close_allowance(): + interrupted = {} + + async def codec(*args, **kwargs): + try: + await asyncio.Future() + except asyncio.CancelledError as error: + interrupted["original"] = error + raise + + conn = Mock(set_type_codec=codec, close=AsyncMock()) + with pytest.raises(TimeoutError): + async with asyncio.timeout(0): + await directory._init(conn) + conn.close.assert_not_awaited() + conn.terminate.assert_called_once_with() + assert interrupted["original"].__notes__ == [ + "directory initialization session closure is unknown" + ] + + +@pytest.mark.parametrize("kind", ["ordinary", "exit"]) +@pytest.mark.parametrize("boundary", ["cancel", "second_cancel", "deadline"]) +async def test_interruption_after_codec_close_starts_keeps_original_and_bound( + monkeypatch, kind, boundary +): + from contextlib import asynccontextmanager + + original = ValueError("codec failed") if kind == "ordinary" else SystemExit(17) + closing, disconnect = asyncio.Event(), asyncio.Event() + healthy = Mock(close=AsyncMock(), terminate=Mock()) + raw = Mock(set_type_codec=AsyncMock(side_effect=original), terminate=Mock()) + budgets = {} + + async def close(*, timeout): # noqa: ASYNC109 — fake driver close API + assert timeout == directory.STATEMENT_TIMEOUT_S + closing.set() + await disconnect.wait() + + raw.close = AsyncMock(side_effect=close) + + class Pool: + loaned = False + + @asynccontextmanager + async def acquire(self): + if self.loaned: + await directory._init(raw) + pytest.fail("failed initializer admitted its raw session") + self.loaned = True + try: + yield healthy + finally: + self.loaned = False + + pool = Pool() + monkeypatch.setattr(directory, "_pool", pool) + + async def acquire(): + async with asyncio.timeout(None) as budget: + budgets["original"] = budget + try: + async with directory._connection(): + pytest.fail("failed initializer returned a loan") + except BaseException as error: # noqa: BLE001 — native originals must not escape the task. + return error + + async with pool.acquire() as loan: + task = asyncio.create_task(acquire()) + started = asyncio.create_task(closing.wait()) + try: + async with asyncio.timeout(1): + await asyncio.wait((task, started), return_when=asyncio.FIRST_COMPLETED) + assert not task.done(), "initializer returned before the close boundary" + budget = budgets["original"] + assert budget.when() is None + end = asyncio.get_running_loop().time() + if boundary == "deadline": + budget.reschedule(end) + else: + task.cancel("first acquisition cancellation") + if boundary == "second_cancel": + task.cancel("second acquisition cancellation") + assert await asyncio.wait_for(task, 0.5) is original + assert asyncio.get_running_loop().time() - end < 0.5 + assert budget.when() == (end if boundary == "deadline" else None) + assert budget.expired() == (boundary == "deadline") + assert original.__notes__ == ["directory initialization session closure is unknown"] + raw.close.assert_awaited_once_with(timeout=directory.STATEMENT_TIMEOUT_S) + raw.terminate.assert_called_once_with() + assert loan is healthy and pool.loaned + healthy.close.assert_not_awaited() + healthy.terminate.assert_not_called() + finally: + disconnect.set() + for pending in (task, started): + if not pending.done(): + pending.cancel() + await asyncio.gather(task, started, return_exceptions=True) diff --git a/tests/unit/test_pg_retirement_settlement.py b/tests/unit/test_pg_retirement_settlement.py index a195fb69b9..4f7bd93365 100644 --- a/tests/unit/test_pg_retirement_settlement.py +++ b/tests/unit/test_pg_retirement_settlement.py @@ -153,7 +153,7 @@ async def owned_command(world): owner = cell_control.Owner() async def opening(): - owner.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) + await owner.connect(statement_timeout_ms=60_000) async def work(): await owner.conn.execute("COMMIT") diff --git a/tests/unit/test_repo_snapshot_contract.py b/tests/unit/test_repo_snapshot_contract.py index 1db7a72183..53410c9762 100644 --- a/tests/unit/test_repo_snapshot_contract.py +++ b/tests/unit/test_repo_snapshot_contract.py @@ -158,7 +158,7 @@ def test_the_accessors_refuse_until_the_registry_is_read(monkeypatch) -> None: org_readable=True, ) monkeypatch.setattr(settings, "code_self_service_enabled", True) - monkeypatch.setattr(repos, "_configured_from_env", lambda: [entry]) + monkeypatch.setattr(repos, "_configured_from_env", lambda _config, _errors: [entry]) monkeypatch.setattr(repos, "registry_repos", lambda: []) monkeypatch.setattr(repos, "_registry_loaded", False) with pytest.raises(repos.RegistryUnread): diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 84bd4db79d..c240249eaa 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -84,7 +84,9 @@ async def first_cell_named(self, name: str) -> directory.Cell | None: row = self.cells.get(name) return row if row is not None and row.is_first else None - async def place(self, org: str, name: str, refusal: Any) -> tuple[str, str]: + async def place( + self, org: str, name: str, refusal: Any, *, actor: str, verify: Any = None + ) -> tuple[str, str]: self._check() if org not in self.placements: row = self.cells[name] # the row as the placement locks it diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 8265ef9620..dd307ece4f 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -970,7 +970,7 @@ def observed(name: str = "eu-1") -> directory.Control: def test_a_live_placement_reads_the_winner_again_under_the_row_lock(store: _Store) -> None: """A batch holds the cell's row lock, not the org's: an org it placed between the live placement's first read and its row lock is answered placed, never judged on the quota, - counted or audited twice.""" + counted twice. Its call still writes the durability audit.""" store.cells["eu-1"].update(state="serving", placements_max=0) reads = 0 real = _Conn.fetchrow @@ -985,9 +985,10 @@ async def _fetchrow(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchrow", _fetchrow) - winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) + winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) assert winner == ("eu-1", "dep-1") and reads == 2 - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] def test_a_live_insert_that_loses_to_the_primary_key_counts_nothing(store: _Store) -> None: @@ -1001,8 +1002,12 @@ async def _fetchval(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchval", _fetchval) - assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) == ("eu-1", "dep-1") - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) == ( + "eu-1", + "dep-1", + ) + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] # The attach lock and the troubleshooting page. @@ -1451,3 +1456,28 @@ def test_native_compact_batch_with_control_envelope_fits_public_handler(client, assert response.status_code == 200, response.text assert response.json()["placed"] == len(batch) assert set(store.placements) == set(batch) + + +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_refuses_operator_envelope_overflow_before_probe_or_writes( + client: TestClient, store: _Store, listener: _Listener, position: str +) -> None: + deployment = str(UUID(int=1)) + store.cells["eu-1"]["deployment_id"] = deployment + current = cell.as_value({**FIELDS, "deployment_id": deployment}) + grown = cell.as_value({**FIELDS, "deployment_id": deployment, "oidc_client_id": ""}) + live = {"org": "", "cell": "eu-1", "fingerprint": grown} + available = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES + grown["fields"]["oidc_client_id"] = "x" * available + body = ( + _begin(deployment_id=deployment, current_fp=current, **{position: grown}) + if position == "next_fp" + else _begin(deployment_id=deployment, current_fp=grown) + ) + response = _call(client, "POST", "attach/begin", body) + assert response.status_code == 409 + assert position in response.json()["error"] + assert "oidc_client_id" in response.json()["error"] + assert store.statements == directory.ATTACH_READ_STATEMENTS + assert listener.asked == [] and store.audit == [] + assert store.cells["eu-1"]["attach_started_at"] is None