diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1abff33cbe..1ed512176d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -901,6 +901,8 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_place.py + tests/integration/test_cell_place_cleanup.py tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py @@ -934,6 +936,34 @@ jobs: bash dev/ci_start_postgres.sh docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby pytest -q tests/integration/test_pg_online_index_durability.py + - name: pytest placement durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-place-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_CELL_PLACE_DRILL: "1" + LOOM_CELL_PLACE_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-place-drill-pg loom-scale-place-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-place-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-place-drill-pg loom-scale-place-drill-pg-standby || true + docker network rm loom-scale-place-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-place-drill-pg loom-scale-place-drill-pg-standby + pytest -q tests/integration/test_cell_place_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index c501cf0e4c..32e81ada9d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -754,13 +754,26 @@ read-only scope, lists any tenancy row the directory does not place. it, unless the cell is the first and its mode reads `recording`, and an org-free command never uses it; a leaked one is stopped by `loom cell limit --closed`, which needs none of the cell's stores, and replaced by `loom cell credential` (below). - `loom cell place --org ` (declared org-free: it writes the org's placement and - tenancy row, which is how an org comes to be held), with the router's operator credential, + `loom cell place --cell --org ` (declared org-free: it writes the org's + placement and tenancy row, which is how an org comes to be held), with the router's operator credential, places an org for a command that writes a new one (a bot, an import), through the router's placement API and in its order, as attach's placement of configured orgs does: it inserts the placement and reads back the winner, writes the cell's tenancy row only if the winner is this cell (otherwise it refuses and writes nothing), and reports success only once the - row is written, so a rerun is idempotent. For an org the directory does not place yet it + row commits synchronously. The CLI holds the attach lock on a direct marked Postgres + session, requires a full enforced local cell state and the full accepted configuration + fingerprint, and checks both again before inserting. A transactional durability message + makes even a no-op insert flush an earlier primary-only tenancy row to the standby. + Opening the session, initial checks and the local transaction through COMMIT each have + a separate client deadline derived from `LOOM_PG_STATEMENT_TIMEOUT_MS`; the directory + request keeps its own transport timeout. Each local phase reserves cleanup inside its + original deadline. On failure the shared control owner closes the exact backend before + cancelling or reaping its work, deferring repeated signals within that same deadline. + Successful exit and failures outside a local phase have a separate cleanup allowance + derived from the same setting. An unknown outcome names rerunning the same command; + the permanent directory winner and idempotent local insert let that rerun finish. The + route checks directory state; a direct operator API caller is trusted to check local + Postgres mode as the CLI does. For an org the directory does not place yet it passes the placement API's gate (below): refused while the directory is unavailable, for a cell the gate does not accept, closed, or when the fingerprint of the CLI's own configuration of the cell (with the deployment id read from the stores it connects to) @@ -789,9 +802,9 @@ read-only scope, lists any tenancy row the directory does not place. serves past a table whose lock it cannot take, logged with the holders of its locks, or one the role cannot alter, logged as such (the third exception under Invariants). A first cell is `recording` until attach sets `enforced` while it runs, so no boot can - have created them: attach creates them itself with the same pass - (`cell_triggers.ensure`) once it has set `enforced`, and the cell goes `serving` only - once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare + have created them: attach creates them with `cell_triggers.ensure_table`, passing + `recording_first=True`, before step 2 and again before setting `enforced`. + The cell goes `serving` only once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare and gets them at its first migrating boot after prepare, which its attach requires the same way. `loom doctor` names a disabled or altered one in any state, a missing one on an `enforced` cell, and a partitioned table with an `org` column as unsupported, since @@ -859,9 +872,12 @@ read-only scope, lists any tenancy row the directory does not place. safe, since the read-back makes it idempotent. For an org the directory already places, a cell's call is checked only for its credential (against the cells' hashes the router caches for `LOOM_ROUTER_DIRECTORY_CACHE_S`, default 5 s) and the cell's - rate, and a `loom cell place` only for the operator credential; both answer the winner - whatever the cell's state, quota or fingerprint, adding no placement (the winner is - public through the lookup anyway); a request refused past its cell's rate answers 503 + rate. An operator's `loom cell place` checks its deployment id and fingerprint against + the requested cell's locked row even for an org already placed there; an org placed + elsewhere answers that winner and the CLI refuses without inserting local tenancy. + Existing placements bypass closed and quota checks. Every successful call writes an + audit row before its synchronous commit, including a retry of an existing winner, so + the returned winner survives failover. A request refused past its cell's rate answers 503 with `Retry-After` (at most a minute), and its retry is admitted by the replica that answers it once the cell's calls there are under the rate. An org the cell already holds needs no call, so serving it never depends on the router. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 12bac9a65b..dd75540091 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -329,6 +329,7 @@ python scripts/codegen/docs_reference.py | `loom cell limit` | Close, reopen or bound a cell to new orgs. | `--cell`, `--closed`, `--open`, `--placements-max`, `--router` | | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | +| `loom cell place` | Place an org in this cell and record its tenancy. | `--cell`, `--org`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | | `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | diff --git a/server/cli/cell.py b/server/cli/cell.py index e43a5a4d07..d583fb48dc 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -349,7 +349,7 @@ def _cmd_remove(args: argparse.Namespace) -> int: print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 try: - answer = asyncio.run(_remove_locked(args, router, row)) + answer = asyncio.run(_remove_locked(args, router)) except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) return 1 @@ -360,7 +360,6 @@ def _cmd_remove(args: argparse.Namespace) -> int: async def _remove_locked( args: argparse.Namespace, router: httpx.URL, - row: dict[str, Any], transport: httpx.AsyncBaseTransport | None = None, ) -> dict[str, Any]: from server.cli import cell_control @@ -463,7 +462,9 @@ def _add_cell_parser(sub: Any) -> None: remove.set_defaults(func=_cmd_remove) from server.cli.cell_attach import add_attach_parser + from server.cli.cell_place import add_place_parser from server.cli.cell_prepare import add_prepare_parser + add_place_parser(csub) add_prepare_parser(csub) add_attach_parser(csub) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index a8a78279fd..821a77448b 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -90,10 +90,19 @@ async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" try: - async with httpx.AsyncClient( - timeout=_HEALTH_TIMEOUT, transport=self.transport - ) as client: + manager = httpx.AsyncClient(timeout=_HEALTH_TIMEOUT, transport=self.transport) + client = await manager.__aenter__() + try: r = await client.get(f"https://{host}/v1/health/deployment") + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + except (httpx.HTTPError, ValueError): + return "" + # A required close failure after a successful read must reach the owner. + await manager.__aexit__(None, None, None) + try: answer = r.json() except (httpx.HTTPError, ValueError): return "" @@ -367,7 +376,11 @@ async def begin(self) -> None: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) # -- the scans --------------------------------------------------------------------- @@ -533,11 +546,16 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.mutate( + answer = await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) + return answer # -- the run ----------------------------------------------------------------------- diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 1966b9e62b..730c36bd42 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import contextlib import json +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -304,16 +306,22 @@ async def request( endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) try: - async with ( - asyncio.timeout(REQUEST_S), - httpx.AsyncClient( + async with asyncio.timeout(REQUEST_S): + manager = httpx.AsyncClient( timeout=cli._OPERATOR_TIMEOUT, trust_env=url.scheme == "https", transport=transport, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) as client, - ): - response = await client.request(method, endpoint, json=body) + ) + client = await manager.__aenter__() + try: + response = await client.request(method, endpoint, json=body) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) except (httpx.HTTPError, TimeoutError) as e: raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py new file mode 100644 index 0000000000..214c79179f --- /dev/null +++ b/server/cli/cell_place.py @@ -0,0 +1,195 @@ +"""Directory-first operator placement, followed by a durable local tenancy row.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import sys +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_pg as apg +from server.cli._ch import run_connected, writes +from server.cli.cell_ch import Refused +from server.cli.cell_control import Owner +from server.core import cell, pg +from server.router.directory import PLACE_MAX_S +from server.settings import settings + +# The directory statement bound plus connection and client cleanup. +REQUEST_S = PLACE_MAX_S + 10.0 # not a setting: derived + + +class PlacementRefused(Exception): + """A placement cannot complete with this configuration or directory winner.""" + + +def _bound_s() -> float: + return settings.pg_statement_timeout_ms / 1000 + 1 + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return "--cell names 1 to 32 of a-z, 0-9, -" + if args.cell != settings.cell: + return f"place runs with the cell's own configuration: set LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return "place needs LOOM_STORAGE_AUTHORITY=postgres and the cell's LOOM_PG_DSN" + if cell.named_org(args.org) is None: + return "--org names a nonempty Postgres org" + return cell.org_name_refusal(args.org) + + +async def _identity(conn: Any, name: str) -> tuple[str, dict[str, Any]]: + """Read the full local cell state and this CLI's accepted configuration.""" + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + state, first = await cell.read_state_if_present(conn) + if state != "enforced" or first != name: + raise PlacementRefused( + f"local cell state is {state!r}, naming {first!r}; place needs enforced {name!r}" + ) + deployment = await conn.fetchval("SELECT deployment_id FROM loom.authority") + if not isinstance(deployment, str) or not deployment: + raise PlacementRefused("the local Postgres records no deployment id") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + mine = cell.fingerprint(deployment) + if own is None or not any( + cell.matches(mine, cell.json_object(own[k])) for k in ("current_fp", "next_fp") + ): + raise PlacementRefused("the CLI's configuration matches no local cell fingerprint") + return deployment, cell.as_value(mine) + + +async def _directory( + router: httpx.URL, name: str, org: str, deployment: str, fingerprint: dict[str, Any] +) -> None: + url = cell_cli.at(router, f"/cells/v1/operator/cells/{name}/place") + manager = httpx.AsyncClient(trust_env=router.scheme == "https") + client = await manager.__aenter__() + try: + answer = await client.post( + url, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, + timeout=httpx.Timeout(REQUEST_S, connect=5.0), + ) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) + if posted is None: + raise PlacementRefused(cell_cli.not_the_router(answer)) + if answer.status_code != 200: + raise PlacementRefused(f"the router refused ({answer.status_code}): {posted['error']}") + if posted.get("cell") != name: + raise PlacementRefused( + f"org {org!r} belongs to cell {posted.get('cell')!r}; no local tenancy row written" + ) + if posted.get("deployment_id") != deployment: + raise PlacementRefused("the router answered another deployment id") + + +async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: + """Hold the operator lock through directory placement and an acknowledged local commit. + Local phases and the complete directory exchange have separate deadlines.""" + owner = Owner(exit_s=_bound_s()) + + async def opening() -> None: + owner.conn = await pg._connect_dedicated( + statement_timeout_ms=settings.pg_statement_timeout_ms + ) + if owner.closed: + await owner.close() + raise PlacementRefused("connection arrived after the placement attempt ended") + try: + await pg._check_deployment(owner.conn) + await pg.init_connection(owner.conn) + except Exception as error: + pg._raise_if_unavailable(error) + raise + + async def initial() -> tuple[str, dict[str, Any]]: + if refusals := await apg.direct_refusals(owner.conn, "place"): + raise Refused(refusals) + await apg.take_lock(owner.conn, "place") + return await _identity(owner.conn, args.cell) + + async def work() -> dict[str, str]: + deployment, fingerprint = await owner.call(initial, total_s=_bound_s()) + await owner.call( + lambda: _directory(router, args.cell, args.org, deployment, fingerprint), + total_s=REQUEST_S, + ) + + async def local() -> None: + conn = owner.conn + tx = conn.transaction() + await tx.start() + if not await apg.lock_held(conn): + raise PlacementRefused("the operator lock was lost") + if await _identity(conn, args.cell) != (deployment, fingerprint): + raise PlacementRefused( + "the local identity or configuration changed during placement" + ) + await conn.execute( + "INSERT INTO loom.tenancy (org) VALUES ($1) ON CONFLICT DO NOTHING", args.org + ) + # A no-op insert must also flush a primary-only predecessor to the standby. + await conn.execute(pg._DURABLE_MESSAGE) + await tx.commit() + + await owner.call(local, total_s=_bound_s()) + return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + return await owner.run(opening, work, opening_s=_bound_s()) + + +def cmd_place(args: argparse.Namespace) -> int: + if reason := _args_problem(args): + print(f"loom cell place: {reason}", file=sys.stderr) + return 2 + router = cell_cli.operator_router(args) + if isinstance(router, str): + print(f"loom cell place: {router}", file=sys.stderr) + return 2 + + async def run(ch: Any) -> int: + try: + result = await place(args, router) + except asyncio.CancelledError: + print( + "loom cell place: stopped (CancelledError); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + raise + except Exception as e: # noqa: BLE001 — an unknown outcome must name its idempotent retry + print( + f"loom cell place: stopped ({type(e).__name__}: {e}); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + return run_connected( + run, org_free=writes("places an org in the directory and records its local tenancy") + ) + + +def add_place_parser(csub: Any) -> None: + p = csub.add_parser("place", help="Place an org in this cell and record its tenancy.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--org", required=True, help="The org to place") + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_place) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 6e94ae1058..517a461840 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,6 +26,7 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets @@ -301,9 +302,11 @@ async def _register( token = settings.router_operator_token url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") deadline = time.monotonic() + args.wait - async with httpx.AsyncClient( + manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport - ) as client: + ) + client = await manager.__aenter__() + try: while True: try: r = await client.post(url, json=body, headers={"Authorization": f"Bearer {token}"}) @@ -318,7 +321,7 @@ async def _register( f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" ) if r.status_code == 200: - return answer + break code = answer.get("code") if code == "BACKEND_UNAVAILABLE": raise Refused( @@ -340,6 +343,14 @@ async def _register( print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) + except BaseException: # Client exit preserves the first request or refusal. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + return answer + def _own_rows( rows: _CellRows, answer: dict[str, Any], every: list[dict[str, Any]] @@ -502,7 +513,11 @@ async def _prepare_locked( try: await _write_nonces(ch, body["nonces"]) answer = await _register(args, router, body, transport) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(ch, body["nonces"]) + raise + else: await _delete_nonces(ch, body["nonces"]) # The processes may have changed while the probe was retried: judge the ones live now. beats = await _read_beats(ch) diff --git a/server/core/cell.py b/server/core/cell.py index 37e20f99cb..fa1a44615e 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -415,10 +415,9 @@ def _unset_reason(view: _View) -> str: ) -# The fingerprint fields a CLI's gate judges: the stores it would write. A CLI verifies -# no token and its fingerprint never reaches the router, so the rest of the servers' -# fingerprint is not its to match (docs/architecture/cells.md, "The CLI declares its -# orgs"). +# An org-declaring CLI's gate checks the stores it would write: it verifies no token +# and sends no fingerprint to the router. Operator placement checks the full fingerprint +# separately (docs/architecture/cells.md, "The CLI declares its orgs"). _STORE_FIELDS = ("deployment_id", "bucket", "bucket_prefix") diff --git a/server/router/app.py b/server/router/app.py index 52c9d3d157..212e87f737 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -14,6 +14,8 @@ - `POST /cells/v1/operator/cells//limit` and `.../credential`: the operator's changes to a cell's row (`loom cell limit`, `loom cell credential`), with the router's operator credential; +- `POST /cells/v1/operator/cells//place`: `loom cell place` places an org with the + operator credential, verifying the requested cell's deployment id and fingerprint; - `POST /cells/v1/operator/cells//register`: `loom cell prepare` registers a cell, once the probe listener (the first cell's at LOOM_ROUTER_UPSTREAM_PROBE_URL, a later cell's at its `probe_url`) answers its deployment id, prepare's nonces and the bucket and @@ -28,9 +30,9 @@ Every refusal of a placement writes nothing. A malformed placement body answers 400; the directory's refusals answer 503, so the cell's request answers 503 and its retry is safe, except a new org longer than `cell.ORG_MAX_BYTES`, which answers 400 `ORG_TOO_LONG` and no -retry places; a credential that names no cell, or not this one, answers 401. An operator -route answers a cell the directory does not hold 404 with the code `NO_SUCH_CELL`, so a 404 -without it is a router of an earlier build. +retry places; a credential that names no cell, or not this one, answers 401. Operator +placement refusals answer 409; the other operator routes answer a cell the directory does +not hold 404 with `NO_SUCH_CELL`, so a 404 without it is a router of an earlier build. """ from __future__ import annotations @@ -302,7 +304,7 @@ def refusal(locked: directory.Cell) -> str: return "" try: - winner, deployment_id = await directory.place(org, name, refusal) + winner, deployment_id = await directory.place(org, name, refusal, actor=f"cell:{name}") except directory.OrgTooLong as e: return _refused(400, e.reason, code="ORG_TOO_LONG") except (directory.PlacementRefused, directory.DirectoryUnavailable) as e: @@ -315,6 +317,41 @@ def refusal(locked: directory.Cell) -> str: log.info("placement of %s: %s (asked by %s)", org, winner, name) return JSONResponse({"cell": winner, "deployment_id": deployment_id}) + async def operator_place(self, request: Request) -> JSONResponse: + """Place with the operator credential. The CLI checks its local Postgres mode; + this route trusts that check and verifies the directory's locked identity.""" + body = await self._operator_body(request, MAX_BODY_BYTES) + if isinstance(body, JSONResponse): + return body + org = cell_state.named_org(body.get("org")) + deployment_id = _text(body, "deployment_id", needed=True) + if ( + set(body) != {"org", "deployment_id", "fingerprint"} + or org is None + or deployment_id is None + ): + return _refused(400, "a placement names org, deployment_id and fingerprint") + if reason := cell_state.org_name_refusal(org): + return _refused(400, reason, code="ORG_TOO_LONG") + name = request.path_params["cell"] + + def verify(locked: directory.Cell) -> str: + if locked.deployment_id != deployment_id: + return "the calling process names another deployment id" + if not fingerprint_matches(locked, body["fingerprint"]): + return f"the calling process's configuration matches no value of {name!r}" + return "" + + try: + winner, deployment = await directory.place( + org, name, placement_refusal, actor="operator", verify=verify + ) + except directory.PlacementRefused as e: + return _refused(409, e.reason) + except directory.DirectoryUnavailable as e: + return _refused(503, e.reason, code="BACKEND_UNAVAILABLE") + return JSONResponse({"org": org, "cell": winner, "deployment_id": deployment}) + async def cell_state(self, request: Request) -> JSONResponse: """`GET /cells/v1/cells/`: the first cell's state, for its own credential only, from this replica's last read of the row (`_state_row`). A first cell in @@ -1460,6 +1497,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: return Starlette( routes=[ Route("/cells/v1/placements", router.place, methods=["POST"]), + Route("/cells/v1/operator/cells/{cell}/place", router.operator_place, methods=["POST"]), Route("/cells/v1/cells/{cell}", router.cell_state, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/limit", router.set_limit, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}/register", router.register, methods=["POST"]), diff --git a/server/router/directory.py b/server/router/directory.py index bdf9f4bdd6..4eca1b74bd 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -12,7 +12,8 @@ placing one org at once agree on the winner, and a retry of the same call answers the same; a batch takes none, since no other cell exists while it runs; - the placements' primary key, as a backstop: every insert is `ON CONFLICT DO NOTHING`, and - only a row it returns is counted and audited. + only a row it returns is counted. Every successful call writes an audit row before + committing, so a duplicate winner waits for synchronous durability too. """ from __future__ import annotations @@ -62,6 +63,10 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) +# New placement: BEGIN, org lock, winner, cell lock, winner, insert, counter, winner, +# audit, COMMIT. The audit also gives duplicate winners a synchronous durability fence. +PLACE_STATEMENTS = 10 +PLACE_MAX_S = _bound_s(PLACE_STATEMENTS) # not a setting: derived # The longest limit/credential change has ten statements: begin, directory lock, row, # other row, WAL, credential check, update, audit, read-back and commit. OPERATOR_CHANGE_MAX_S = _bound_s(10) # not a setting: derived from the bound above @@ -674,39 +679,49 @@ async def first_cell_named(cell_name: str) -> Cell | None: return _cell(row) if row else None -async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: - """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) - that holds it. An org placed already answers its cell, whatever the cell's state. - Otherwise the cell's row is locked, the winner read again (an attach batch holds the row - lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it - waits for the placement and the quota holds across router replicas; a refused placement - raises PlacementRefused and writes nothing, and a new org name longer than - `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as - any placed org does).""" +async def place( + org: str, + cell_name: str, + refusal: Callable[[Cell], str], + *, + actor: str, + verify: Callable[[Cell], str] | None = None, +) -> tuple[str, str]: + """The permanent winner of `org`, with every successful call audited and committed + synchronously. `verify` judges the requested cell's locked identity even for a winner + already there; `refusal` and quota judge new placements only. A winner elsewhere needs + neither check. The org lock and cell row lock serialize live and attach placements.""" async with _reading(), _connection() as conn, conn.transaction(): - # Placements of one org run one at a time, whichever cell asks: the later one - # reads the earlier one's winner. await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) row = await conn.fetchrow(_WINNER, org) - if row is None: - _refuse_long([org]) + if row is None or (verify is not None and row["cell"] == cell_name): + if row is None: + _refuse_long([org]) locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) if locked is None: raise PlacementRefused(f"cell {cell_name!r} is not in the directory") - # An attach batch of this cell, which holds the row lock and not the org's, may - # have placed the org since the first read: it is answered as placed. + candidate = _cell(locked) + # An attach batch holds this row's lock rather than the org's. row = await conn.fetchrow(_WINNER, org) + if (row is None or row["cell"] == cell_name) and verify is not None: + if reason := verify(candidate): + raise PlacementRefused(reason) if row is None: - await _place_one(conn, org, cell_name, _cell(locked), refusal) + await _place_one(conn, org, cell_name, candidate, refusal) row = await conn.fetchrow(_WINNER, org) - return str(row["cell"]), str(row["deployment_id"]) + winner = str(row["cell"]), str(row["deployment_id"]) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + actor, + {"org": org, "cell": winner[0]}, + ) + return winner async def _place_one( conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] ) -> None: - """Judge the locked row (`refusal`, then its quota) and insert the placement; its count - and audit row only when the insert wrote it.""" + """Judge the locked row and insert the placement, counted only when written.""" if reason := refusal(candidate): raise PlacementRefused(reason) quota = candidate.placements_max @@ -723,11 +738,6 @@ async def _place_one( await conn.execute( "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name ) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", - f"cell:{cell_name}", - {"org": org, "cell": cell_name}, - ) def _refuse_long(orgs: Iterable[str]) -> None: @@ -849,12 +859,16 @@ def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> _PLACEMENT_ORG_MAX = 6 * cell_state.ORG_MAX_BYTES # not a setting: derived -def _placement_refusal(cell: str, name: str, value: dict[str, Any]) -> str: - """Why a fingerprint is too large for the cell's live placements ('' when it is not): a - placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's - MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" +def _placement_refusal(cell: str, deployment_id: str, name: str, value: dict[str, Any]) -> str: + """Reserve both live and operator placement bodies with the longest escaped org.""" + # Native callers materialize known defaults; retain unknown stored fields too. + value = {**value, "fields": {**cell_state.DEFAULTS, **value["fields"]}} # The default encoding is the longest: spaces after separators, non-ASCII escaped. - size = len(json.dumps({"org": "", "cell": cell, "fingerprint": value})) + _PLACEMENT_ORG_MAX + envelopes = ( + {"org": "", "cell": cell, "fingerprint": value}, + {"org": "", "deployment_id": deployment_id, "fingerprint": value}, + ) + size = max(len(json.dumps(body)) for body in envelopes) + _PLACEMENT_ORG_MAX if size <= MAX_BODY_BYTES: return "" sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) @@ -880,7 +894,7 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: if value is None: continue if reason := _fingerprint_refusal(found, name, value) or _placement_refusal( - row.cell, name, value + row.cell, row.deployment_id, name, value ): return reason # Compared as JSON, not as Python values, where `false == 0` and `1 == 1.0`: a rerun diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index e14eab1220..14d469d743 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -468,7 +468,7 @@ async def wait_for_lock(key): "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) # The placement has its org lock and is queued at the cell row before claim. for _ in range(100): @@ -502,7 +502,7 @@ async def wait_for_lock(key): "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) await wait_for_lock(directory._PLACE_LOCK) await cell_control.backout(_owner(local_admin), call, original) @@ -661,7 +661,9 @@ async def call(path, body): ) # Actually attempt placement before the acknowledged close returns to cleanup. try: - await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + await directory.place( + "between-close-and-cleanup", "first", app.placement_refusal, actor="cell:first" + ) except directory.PlacementRefused as exc: placement_error = str(exc) return answer diff --git a/tests/integration/test_cell_place.py b/tests/integration/test_cell_place.py new file mode 100644 index 0000000000..7024024002 --- /dev/null +++ b/tests/integration/test_cell_place.py @@ -0,0 +1,238 @@ +"""Operator placement on two cells' databases and a real directory, without memory rows.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import os +import time + +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.router import app, directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database, own_database + +OPERATOR = "lco_" + "3" * 128 +ROUTER = httpx.URL("http://127.0.0.1:7901") + + +@pytest.fixture +async def stores(monkeypatch): + admin = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, dsn = await create_database(admin) + async with contextlib.AsyncExitStack() as stack: + stack.push_async_callback(drop_database, admin, name) + first = await stack.enter_async_context(own_database()) + second = await stack.enter_async_context(own_database()) + monkeypatch.setattr(directory, "_pool", None) + stack.push_async_callback(directory.close) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + await directory.migrate() + pg.expect_deployment(None) + router_app = app.build_app(app.Router()) + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kw: real_client(transport=httpx.ASGITransport(app=router_app), **kw), + ) + targets = {"first": first, "second": second} + try: + for cell_name, cell_dsn in targets.items(): + await _initialize(cell_dsn, cell_name) + yield targets + finally: + pg.expect_deployment(None) + await pg.close_pool() + await directory.close() + + +async def _initialize(dsn, name): + import asyncpg + + fields = cell.fingerprint(f"dep-{name}") + conn = await asyncpg.connect(dsn) + try: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + f"dep-{name}", + ) + await conn.execute( + "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", name == "first", name + ) + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6)", + name, + name == "first", + f"{name}.example.test", + hashlib.sha256(name.encode()).hexdigest(), + f"dep-{name}", + cell.as_value(fields), + ) + + +async def _local(stores, monkeypatch, name, org="acme"): + monkeypatch.setattr(settings, "cell", name) + monkeypatch.setattr(settings, "pg_dsn", stores[name]) + return await cell_place.place(argparse.Namespace(cell=name, org=org), ROUTER) + + +async def _tenancy(dsn): + import asyncpg + + conn = await asyncpg.connect(dsn) + try: + return [r["org"] for r in await conn.fetch("SELECT org FROM loom.tenancy ORDER BY org")] + finally: + await conn.close() + + +async def test_operator_place_retries_and_refuses_a_different_cell( + stores, monkeypatch, record_property +): + started = time.perf_counter() + first = await _local(stores, monkeypatch, "first") + record_property("local_first_place_ms", (time.perf_counter() - started) * 1000) + assert first == {"org": "acme", "cell": "first", "deployment_id": "dep-first"} + started = time.perf_counter() + assert await _local(stores, monkeypatch, "first") == first + record_property("local_duplicate_place_ms", (time.perf_counter() - started) * 1000) + assert await _tenancy(stores["first"]) == ["acme"] + with pytest.raises(cell_place.PlacementRefused, match="belongs to cell 'first'"): + await _local(stores, monkeypatch, "second") + assert await _tenancy(stores["second"]) == [] + async with directory._connection() as conn: + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'first'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 3 + ) + + +async def test_two_cells_racing_one_org_have_one_winner_and_count_once(stores): + async def place(name): + fields = cell.as_value(cell.fingerprint(f"dep-{name}")) + async with httpx.AsyncClient() as client: + return await client.post( + str(ROUTER.copy_with(path=f"/cells/v1/operator/cells/{name}/place")), + json={"org": "raced", "deployment_id": f"dep-{name}", "fingerprint": fields}, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + a, b = await asyncio.gather(place("first"), place("second")) + assert a.status_code == b.status_code == 200 + assert a.json() == b.json() + async with directory._connection() as conn: + assert await conn.fetchval("SELECT sum(placed) FROM loom_router.cells") == 1 + assert await conn.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + assert await _tenancy(stores["first"]) == await _tenancy(stores["second"]) == [] + + +@pytest.mark.parametrize("limit", ["closed = true", "placements_max = 0"]) +async def test_new_closed_or_full_cell_writes_neither_store(stores, monkeypatch, limit): + async with directory._connection() as conn: + await conn.execute(f"UPDATE loom_router.cells SET {limit} WHERE cell = 'first'") + with pytest.raises(cell_place.PlacementRefused, match="router refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert await directory.lookup("acme") is None + + +async def test_failed_local_insert_resumes_and_existing_winner_ignores_closed_quota( + stores, monkeypatch +): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute( + "CREATE FUNCTION loom.fail_place() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'local insertion refused'; END $$" + ) + await conn.execute( + "CREATE TRIGGER fail_place BEFORE INSERT ON loom.tenancy FOR EACH ROW EXECUTE FUNCTION loom.fail_place()" + ) + with pytest.raises(asyncpg.RaiseError, match="local insertion refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert (await directory.lookup("acme"))[0] == "first" + await conn.execute("DROP TRIGGER fail_place ON loom.tenancy") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +async def test_first_recording_refuses_and_enforced_attaching_succeeds(stores, monkeypatch): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + with pytest.raises(cell_place.PlacementRefused, match="needs enforced"): + await _local(stores, monkeypatch, "first") + assert await directory.lookup("acme") is None + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET state = 'attaching' WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +@pytest.mark.parametrize("changed", ["mode", "fingerprint"]) +async def test_changed_local_state_after_directory_call_refuses_insert( + stores, monkeypatch, changed +): + import asyncpg + + real = cell_place._directory + + async def change_after_call(*args): + await real(*args) + conn = await asyncpg.connect(stores["first"]) + try: + if changed == "mode": + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + else: + fields = cell.fingerprint("dep-first") + fields["oidc_client_id"] = "changed" + await conn.execute( + "UPDATE loom.cell_self SET current_fp = $1::jsonb", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + + monkeypatch.setattr(cell_place, "_directory", change_after_call) + with pytest.raises(cell_place.PlacementRefused): + await _local(stores, monkeypatch, "first") + assert (await directory.lookup("acme"))[0] == "first" + assert await _tenancy(stores["first"]) == [] diff --git a/tests/integration/test_cell_place_cleanup.py b/tests/integration/test_cell_place_cleanup.py new file mode 100644 index 0000000000..3bbb5acc07 --- /dev/null +++ b/tests/integration/test_cell_place_cleanup.py @@ -0,0 +1,106 @@ +"""Placement cleanup observes the actual marked backend through interrupted work.""" + +from __future__ import annotations + +import asyncio +import contextlib +import time + +import asyncpg +import pytest + +from server.cli import cell_place +from server.core import pg +from tests.integration import test_cell_place as placement + + +@pytest.fixture +async def stores(monkeypatch): + async for value in placement.stores.__wrapped__(monkeypatch): + yield value + + +async def waiting(observer, pid): + async with asyncio.timeout(3): + while True: + if await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event='PgSleep')", + pid, + ): + return + await asyncio.sleep(0.01) + + +async def assert_released(observer, pid): + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +@pytest.mark.parametrize("interrupt", ["deadline", "cancel", "second_cancel"]) +async def test_placement_owner_closes_before_cancelling_actual_rollback( + stores, monkeypatch, interrupt +): + # Only the exact ROLLBACK wire operation is delayed. The transaction context, + # sleeping query, session lock, observer and close are the installed real driver. + original_identity, execute = cell_place._identity, asyncpg.Connection.execute + known = asyncio.Event() + captured = {} + queries = [] + + async def stalled_rollback(conn, sql, *args, **kwargs): + if conn is captured.get("conn") and sql == "ROLLBACK;": + query = asyncio.create_task(execute(conn, "SELECT pg_sleep(10)")) + queries.append(query) + while not query.done(): + try: + await asyncio.shield(query) + except asyncio.CancelledError: + continue + query.result() + return await execute(conn, sql, *args, **kwargs) + + async def identity(conn, name): + await original_identity(conn, name) + captured["conn"] = conn + known.set() + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(193789)") + raise ValueError("placement body failed before rollback") + + monkeypatch.setattr(cell_place, "_identity", identity) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + observer = await asyncpg.connect(stores["first"]) + task = asyncio.create_task(placement._local(stores, monkeypatch, "first")) + try: + await asyncio.wait_for(known.wait(), 3) + conn = captured["conn"] + pid = conn.get_server_pid() + await waiting(observer, pid) + started = time.monotonic() + if interrupt != "deadline": + task.cancel() + if interrupt == "second_cancel": + # A second signal arrives while the same operation is still active. + await asyncio.sleep(0) + task.cancel() + done, _ = await asyncio.wait({task}, timeout=0.8) + assert done, "placement must close its backend before waiting for rollback unwind" + with pytest.raises(TimeoutError if interrupt == "deadline" else asyncio.CancelledError): + await task + assert time.monotonic() - started < 0.8 + await assert_released(observer, pid) + finally: + # Exact owned test resources only; even a deliberately failing old owner + # cannot leave its query or scratch database alive after the regression. + if "conn" in captured and not captured["conn"].is_closed(): + with contextlib.suppress(Exception): + await captured["conn"].close(timeout=1) + if not task.done(): + task.cancel() + await asyncio.gather(task, *queries, return_exceptions=True) + await observer.close(timeout=1) + pg.expect_deployment(None) diff --git a/tests/integration/test_cell_place_durability.py b/tests/integration/test_cell_place_durability.py new file mode 100644 index 0000000000..1d788f47d6 --- /dev/null +++ b/tests/integration/test_cell_place_durability.py @@ -0,0 +1,310 @@ +"""Placement retries survive promotion of a private physical standby. + +Only the dedicated CI drill enables this file. Its pair is separate from the +integration server's stores because it pauses replication and stops the primary. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import os +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlsplit + +import asyncpg +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.core.schema import pg_migrate +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-place-drill-pg" +STANDBY = PRIMARY + "-standby" +ORG = "placement-durability" +DEPLOYMENT = "placement-drill" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +@dataclass +class Drill: + observer: Any + directory_dsn: str + local_dsn: str + tasks: list[asyncio.Task] = field(default_factory=list) + paused: bool = False + + def task(self, work: Any) -> asyncio.Task: + task = asyncio.create_task(work) + self.tasks.append(task) + return task + + async def pause(self) -> None: + await _docker("pause", STANDBY) + self.paused = True + + async def resume(self) -> None: + await _docker("unpause", STANDBY) + self.paused = False + + async def waiting(self, task: asyncio.Task, application: str) -> int: + async with asyncio.timeout(5): + while True: + assert not task.done(), "placement acknowledged before the standby resumed" + pid = await self.observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + application, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.02) + + async def seed(self, dsn: str, sql: str, *, marked: bool = False) -> None: + """Cancel COMMIT only after it is primary-visible and waiting for standby flush.""" + application = f"placement-seed-{len(self.tasks)}" + conn = await asyncpg.connect( + dsn, + timeout=5, + command_timeout=5, + server_settings={ + "application_name": application, + "synchronous_commit": "on", + **(pg.marker_settings() if marked else {}), + }, + ) + try: + await conn.execute("BEGIN") + await conn.execute(sql) + commit = self.task(conn.execute("COMMIT")) + await self.waiting(commit, application) + commit.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(commit, 5) + finally: + conn.terminate() + + +@pytest.fixture +async def drill(monkeypatch): + if not os.environ.get("LOOM_CELL_PLACE_DRILL"): + pytest.skip("LOOM_CELL_PLACE_DRILL not set: requires its disposable physical standby") + assert os.environ["LOOM_CELL_PLACE_DRILL"] == "1" + dsn = os.environ["LOOM_CELL_PLACE_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + state = None + try: + # Refuse to disrupt a container unless the explicit DSN identifies this exact pair. + identifier = str( + await observer.fetchval("SELECT system_identifier FROM pg_control_system()") + ) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + _, directory_dsn = await create_database(dsn) + _, local_dsn = await create_database(dsn) + state = Drill(observer, directory_dsn, local_dsn) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(settings, "router_pg_dsn", directory_dsn) + monkeypatch.setattr(settings, "pg_dsn", local_dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_statement_timeout_ms", 10_000) + previous_expected = pg._expected + previous_orgs = cell._code_orgs + pg.expect_deployment(None) + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + try: + async with asyncio.timeout(60): + await _initialize(state) + yield state + finally: + pg.expect_deployment(previous_expected) + cell._code_orgs = previous_orgs + finally: + if state is not None: + if state.paused: + await state.resume() + for task in state.tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(15): + await asyncio.gather(*state.tasks, return_exceptions=True) + await directory.close() + await pg.close_pool() + observer.terminate() + + +async def _initialize(drill: Drill) -> None: + @contextlib.asynccontextmanager + async def connect(): + conn = await asyncpg.connect(drill.local_dsn, command_timeout=10) + try: + yield conn + finally: + conn.terminate() + + await pg_migrate.migrate(connect=connect) + await directory.migrate() + fingerprint = cell.as_value(cell.fingerprint(DEPLOYMENT)) + async with connect() as conn: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + DEPLOYMENT, + ) + await conn.execute("INSERT INTO loom.cell_first (is_first, cell) VALUES (true, 'first')") + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", json.dumps(fingerprint) + ) + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ('first', true, 'first.example.test', 'serving', 'enforced', $1, $2, $3)", + "3" * 64, + DEPLOYMENT, + fingerprint, + ) + + +async def _directory_retry(drill: Drill) -> None: + await drill.pause() + await drill.seed( + drill.directory_dsn, + "INSERT INTO loom_router.placements (org, cell) VALUES ('placement-durability', 'first'); " + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = 'first'", + ) + assert await directory.lookup(ORG) == ("first", "first.example.test") + retry = drill.task(directory.place(ORG, "first", lambda _: "", actor="operator")) + await drill.waiting(retry, "loom-router") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == ("first", DEPLOYMENT) + + +async def _local_retry(drill: Drill, monkeypatch) -> None: + async def verified_directory(router, name, org, deployment, fingerprint): + # The directory retry above already acknowledged this winner's physical flush. + assert (name, org, deployment) == ("first", ORG, DEPLOYMENT) + assert await directory.lookup(org) == (name, "first.example.test") + + monkeypatch.setattr(cell_place, "_directory", verified_directory) + await drill.pause() + await drill.seed( + drill.local_dsn, + "INSERT INTO loom.tenancy (org) VALUES ('placement-durability')", + marked=True, + ) + assert ( + await _sql( + PRIMARY, + urlsplit(drill.local_dsn).path.removeprefix("/"), + "SELECT org FROM loom.tenancy", + ) + == ORG + ) + args = argparse.Namespace(cell="first", org=ORG) + cancelled = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + pid = await drill.waiting(cancelled, "loom") + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(cancelled, cell_place._bound_s() + 1) + async with asyncio.timeout(5): + while True: + if not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.02) + assert not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ) + retry = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + await drill.waiting(retry, "loom") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == { + "org": ORG, + "cell": "first", + "deployment_id": DEPLOYMENT, + } + + +@pytest.mark.recovery +@pytest.mark.destructive +async def test_duplicate_placement_survives_physical_standby_promotion(drill, monkeypatch): + async with asyncio.timeout(120): + await _directory_retry(drill) + await _local_retry(drill, monkeypatch) + await directory.close() + await pg.close_pool() + drill.observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + directory_db = urlsplit(drill.directory_dsn).path.removeprefix("/") + local_db = urlsplit(drill.local_dsn).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT p.org, p.cell, c.deployment_id, c.placed FROM loom_router.placements p " + "JOIN loom_router.cells c USING (cell)", + ) + == f"{ORG}|first|{DEPLOYMENT}|1" + ) + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT actor, action, detail->>'org', detail->>'cell' FROM loom_router.audit", + ) + == f"operator|place|{ORG}|first" + ) + assert await _sql(STANDBY, local_db, "SELECT org FROM loom.tenancy") == ORG diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index 41f8d343c6..87ecb5cd17 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -127,7 +127,8 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, org ) racing = asyncio.gather( - directory.place(org, a, _never), directory.place(org, b, _never) + directory.place(org, a, _never, actor="operator"), + directory.place(org, b, _never, actor="operator"), ) # Both placements wait on the org's lock before it is released. for _ in range(100): @@ -151,10 +152,11 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT actor FROM loom_router.audit WHERE action = 'place' AND detail->>'org' = $1", org, ) - assert [r["actor"] for r in audit] == [f"cell:{winners[0][0]}"] + assert [r["actor"] for r in audit] == ["operator", "operator"] # Ten concurrent placements in a cell with room for three: three are written. results = await asyncio.gather( - *(directory.place(f"{q}-{i}", q, _never) for i in range(10)), return_exceptions=True + *(directory.place(f"{q}-{i}", q, _never, actor="operator") for i in range(10)), + return_exceptions=True, ) assert sum(isinstance(r, tuple) for r in results) == 3 assert sum(isinstance(r, directory.PlacementRefused) for r in results) == 7 @@ -163,10 +165,13 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: assert held == 3 # An org placed already answers its cell, whatever the quota or the cell's state. placed = next(f"{q}-{i}" for i, r in enumerate(results) if isinstance(r, tuple)) - assert await directory.place(placed, q, lambda _r: "closed") == (q, f"dep-{q}") + assert await directory.place(placed, q, lambda _r: "closed", actor="operator") == ( + q, + f"dep-{q}", + ) # A refusal judged on the locked row writes nothing. with pytest.raises(directory.PlacementRefused, match="closed"): - await directory.place(f"new-{a}", a, lambda _r: "closed") + await directory.place(f"new-{a}", a, lambda _r: "closed", actor="operator") assert await directory.lookup(f"new-{a}") is None # Routing reads every cell with its upstream address ('' until attach writes it). cells, _aliases = await directory.routing() @@ -796,7 +801,9 @@ async def _held(*a: Any) -> None: conn = await asyncpg.connect(own_dsn) try: async with asyncio.timeout(30): - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await inserted.wait() batch = asyncio.ensure_future( directory.attach_place(name, dep, ["acme", "beta"], expected=await _observed(name)) @@ -846,12 +853,17 @@ async def hold(conn, found): directory.attach_place(name, dep, ["acme"], expected=await _observed(name)) ) await written.wait() - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await batch == (1, 1) assert await live == (name, dep) - assert await _placements_of(conn, name) == (1, [("operator", "acme")]) + assert await _placements_of(conn, name) == ( + 1, + [("operator", "acme"), (f"cell:{name}", "acme")], + ) finally: release.set() await conn.close() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 1e7acd696a..b19a76458a 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", - "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", + "usage: loom cell [-h] {credential,limit,which,remove,place,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,place,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -455,6 +455,10 @@ "usage: loom cell remove [-h] --cell CELL [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL The cell to remove (its LOOM_CELL) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell place": [ + "usage: loom cell place [-h] --cell CELL --org ORG [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --org ORG The org to place --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom cell prepare": [ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 3bc4f24ee5..1607d709eb 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12811,6 +12811,99 @@ } ] }, + "place": { + "prog": "loom cell place", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_place:cmd_place" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "org", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--org" + ], + "dest": "org", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The org to place", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] + }, "prepare": { "prog": "loom cell prepare", "usage": null, @@ -13163,6 +13256,7 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", + "place": "loom cell place", "prepare": "loom cell prepare", "attach": "loom cell attach" }, @@ -13187,6 +13281,11 @@ "help": "Remove the directory row of a cell attaching with no placement.", "metavar": "remove" }, + { + "dest": "place", + "help": "Place an org in this cell and record its tenancy.", + "metavar": "place" + }, { "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 64359d1293..6722244a2a 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -143,7 +143,7 @@ def test_cell_control_migrations_are_classified(): ("register", "INSERT INTO", "loom_router.audit"): 1, ("_place_one", "INSERT INTO", "loom_router.placements"): 1, ("_place_one", "UPDATE", "loom_router.cells"): 1, - ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("place", "INSERT INTO", "loom_router.audit"): 1, ("_audit", "INSERT INTO", "loom_router.audit"): 1, }, } @@ -493,9 +493,9 @@ def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): # Operator reads may audit, but cannot bypass control guards for a state write. assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) - assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( - found.functions["place"] - ) + assert "_place_one(conn, org, cell_name, candidate, refusal)" in calls(found.functions["place"]) + assert "_cell(locked)" in calls(found.functions["place"]) + assert "verify(candidate)" in calls(found.functions["place"]) assert "refusal(candidate)" in calls(found.functions["_place_one"]) for name in ("set_limit", "rotate_credential"): assert not any(call.startswith("claim(") for call in calls(found.functions[name])) diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py new file mode 100644 index 0000000000..c9b61bd146 --- /dev/null +++ b/tests/unit/test_cell_native_http_first_error.py @@ -0,0 +1,259 @@ +"""Native HTTP clients preserve the first outcome through required exit.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.settings import settings + +_CLIENT = httpx.AsyncClient +_BOUNDARIES = ("control", "health", "register") +FIRST_ERRORS = ( + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) +EXIT_ERRORS = ( + RuntimeError, + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) + + +@pytest.fixture +def exchange(monkeypatch): + state = SimpleNamespace( + first=None, + second=None, + phase="request", + close_phase="await", + events=[], + exit_args=None, + entered=asyncio.Event(), + block=False, + answer={"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"}, + status=200, + close_entered=asyncio.Event(), + close_block=False, + ) + + async def handler(request): + state.events.append("request") + state.entered.set() + if state.block: + try: + await asyncio.Future() + except asyncio.CancelledError as error: + state.first = error + raise + if state.first is not None: + raise state.first + return httpx.Response(state.status, json=state.answer) + + class Client(_CLIENT): + async def __aenter__(self): + state.events.append("enter") + if state.phase == "enter": + raise state.first + return await super().__aenter__() + + def __aexit__(self, *args): + state.exit_args = args + state.events.append("exit-call") + if state.close_phase == "construct" and state.second is not None: + raise state.second + + async def close(): + state.events.append("exit-await") + state.close_entered.set() + if state.close_block: + await asyncio.Future() + await super(Client, self).__aexit__(*args) + if state.second is not None: + raise state.second + + return close() + + def client(**kwargs): + state.events.append("construct") + if state.phase == "construct": + raise state.first + kwargs["transport"] = httpx.MockTransport(handler) + return Client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return state + + +async def invoke(boundary): + url = httpx.URL("https://router.test") + if boundary == "control": + return await cell_control.request(url, "eu-1", None, "/claim", {}) + if boundary == "health": + return await cell_attach.Router(url, "eu-1", None).health("eu-1.example.test") + return await cell_prepare._register(argparse.Namespace(cell="eu-1", wait=0), url, {}, None) + + +async def observed(callback): + # Native fatal values must stay in this task rather than reach the event loop. + try: + return "value", await callback() + except BaseException as error: # noqa: BLE001 — compare original native failure objects. + return "error", error + + +def assert_first(boundary, state, outcome): + kind, result = outcome + if boundary == "health" and isinstance(state.first, (httpx.HTTPError, ValueError)): + assert (kind, result) == ("value", "") + elif (boundary == "control" and isinstance(state.first, (httpx.HTTPError, TimeoutError))) or ( + boundary == "register" + and state.phase == "request" + and isinstance(state.first, httpx.HTTPError) + ): + expected = cell_prepare.Refused if boundary == "register" else cell_control.Refused + assert kind == "error" and isinstance(result, expected) + assert type(state.first).__name__ in str(result) + else: + assert kind == "error" and result is state.first + + +def assert_exit_first(boundary, state): + first = state.exit_args[1] + if boundary == "register" and isinstance(state.first, httpx.HTTPError): + assert isinstance(first, cell_prepare.Refused) + else: + assert first is state.first + + +def assert_close_failure(boundary, state, outcome): + kind, result = outcome + if boundary == "control" and isinstance(state.second, (httpx.HTTPError, TimeoutError)): + assert kind == "error" and isinstance(result, cell_control.Refused) + else: + assert kind == "error" and result is state.second + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = "dep-eu-1" if boundary == "health" else exchange.answer + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +@pytest.mark.parametrize("semantic", ("malformed", "backend", "probe", "refusal")) +async def test_registration_selected_refusal_survives_exit( + monkeypatch, exchange, second_kind, close_phase, semantic +): + selected = cell_prepare.Refused("registration-first") + monkeypatch.setattr(cell_prepare, "Refused", lambda message: selected) + exchange.status = 200 if semantic == "malformed" else 503 + exchange.answer = ( + {"unexpected": True} + if semantic == "malformed" + else { + "error": "registration-first", + "code": {"backend": "BACKEND_UNAVAILABLE", "probe": "PROBE_BUSY", "refusal": "OTHER"}[ + semantic + ], + } + ) + exchange.second = second_kind("exit-second") + exchange.close_phase = close_phase + kind, result = await observed(lambda: invoke("register")) + assert kind == "error" and result is selected + assert exchange.exit_args[1] is selected diff --git a/tests/unit/test_cell_native_nonce_first_error.py b/tests/unit/test_cell_native_nonce_first_error.py new file mode 100644 index 0000000000..94254c3d96 --- /dev/null +++ b/tests/unit/test_cell_native_nonce_first_error.py @@ -0,0 +1,269 @@ +"""Native callers retain their first outcome while deleting their probe nonces.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.core import leases, object_store, pg +from server.settings import settings +from tests.unit import test_cell_native_http_first_error as native_http +from tests.unit.test_cell_attach import FakeConn +from tests.unit.test_cell_control import snapshot + +exchange = native_http.exchange +observed = native_http.observed +FIRST_ERRORS = native_http.FIRST_ERRORS +CALLERS = ("prepare", "begin", "serving") +FATAL_ERRORS = (asyncio.CancelledError, SystemExit, KeyboardInterrupt) +CLEANUP_PHASES = ("postgres", "clickhouse", "bucket") +URL = httpx.URL("http://router.test") + + +@pytest.fixture +def nonce_edges(monkeypatch, exchange): + state = SimpleNamespace( + secondary=None, + phase="postgres", + write_first=False, + events=[], + normalized=[], + cleanup_entered=asyncio.Event(), + cleanup_block=False, + ) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": "bucket"} + + async def deletion(phase): + state.events.append(phase) + if phase == state.phase: + state.cleanup_entered.set() + if state.cleanup_block: + await asyncio.Future() + if state.secondary is not None: + raise state.secondary + + async def postgres(*args): + await deletion("postgres") + + async def clickhouse(*args): + assert args[0] == leases.LOCK_RELEASE + await deletion("clickhouse") + + async def read_object(location): + return b"bucket" + + async def bucket(*args): + await deletion("bucket") + + async def write(*args): + if state.write_first: + raise exchange.first + + monkeypatch.setattr(pg, "execute", postgres) + monkeypatch.setattr(object_store, "root_prefix", lambda: "") + monkeypatch.setattr(object_store, "read", read_object) + monkeypatch.setattr(object_store, "delete", bucket) + monkeypatch.setattr(cell_prepare, "_new_nonces", lambda: nonces.copy()) + monkeypatch.setattr(cell_prepare, "_write_nonces", write) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "bucket-test") + state.ch = SimpleNamespace(execute=clickhouse) + return state + + +@pytest.fixture +def native_call(monkeypatch, nonce_edges, exchange): + state = nonce_edges + row = {**snapshot(), "is_first": False, "state": "attaching", "mode": "enforced"} + exchange.answer = row.copy() + conn = FakeConn() + rows = SimpleNamespace(deployment_id="dep", first=None, tenancy_rows=0) + monkeypatch.setattr(cell_prepare, "_read_rows", AsyncMock(return_value=rows)) + monkeypatch.setattr(cell_prepare, "_read_beats", AsyncMock(return_value=[])) + monkeypatch.setattr(cell_prepare, "_beat_fields", lambda *args: [{}]) + monkeypatch.setattr(cell_prepare, "_state_rows", lambda rows: 0) + monkeypatch.setattr(cell_prepare, "_prefix_objects", AsyncMock(return_value=0)) + monkeypatch.setattr(cell_prepare, "_answer_problem", lambda *args: "") + monkeypatch.setattr(cell_prepare, "_own_rows", lambda *args: ({}, None)) + state.write_rows = AsyncMock(return_value=[]) + monkeypatch.setattr(cell_prepare, "_write_rows", state.write_rows) + monkeypatch.setattr(cell_control, "claim", AsyncMock(side_effect=lambda conn, call, row: row)) + args = argparse.Namespace( + cell="first", + wait=0, + host="first.example.test", + credential_sha256="hash", + probe_url="", + ch_allow_user=[], + ch_allow_user_directory=[], + ch_allow_object=[], + ch_allow_replica=[], + pg_allow_role=[], + ) + + async def mutate(path, body): + return await cell_control.request(URL, "first", None, path, body) + + run = SimpleNamespace( + ch=state.ch, args=args, row=row, own_fp=({}, None), keys_fp="key", mutate=mutate + ) + + async def invoke(caller, owned): + owner = cell_control.Owner(exit_s=0.5) + direct = SimpleNamespace( + conn=conn, transaction=lambda work: work(), call=lambda work, **kw: work() + ) + + async def work(): + if caller == "prepare": + return await cell_prepare._prepare_locked( + state.ch, args, URL, None, owner if owned else direct + ) + return await getattr(cell_attach.Run, caller)(run) + + async def opening(): + owner.conn = conn + + return await owner.run(opening, work, opening_s=0.5) if owned else await work() + + state.conn = conn + state.invoke = invoke + return state + + +@pytest.fixture(autouse=True) +def selected_normalization(monkeypatch, nonce_edges): + original_prepare = cell_prepare.Refused + original_control = cell_control._refused + + class PrepareRefused(original_prepare): + def __init__(self, message): + super().__init__(message) + nonce_edges.normalized.append(self) + + def control_refused(message): + selected = original_control(message) + nonce_edges.normalized.append(selected) + return selected + + monkeypatch.setattr(cell_prepare, "Refused", PrepareRefused) + monkeypatch.setattr(cell_control, "_refused", control_refused) + + +def assert_first(state, exchange, caller, first_kind, owned, outcome): + kind, result = outcome + assert kind == "error" + normalized = not state.write_first and ( + (caller == "prepare" and issubclass(first_kind, httpx.HTTPError)) + or (caller != "prepare" and issubclass(first_kind, (httpx.HTTPError, TimeoutError))) + ) + if normalized: + assert state.normalized and result is state.normalized[0] + elif owned and first_kind is asyncio.CancelledError: + # asyncio reconstructs cancellation when Owner retrieves its child Task. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert state.write_rows.await_count == 0 + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +@pytest.mark.parametrize("http_exit_failure", (False, True)) +async def test_http_first_outcome_survives_native_nonce_cleanup( + native_call, exchange, caller, owned, first_kind, cleanup_kind, phase, http_exit_failure +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-second") if cleanup_kind is not None else None + exchange.first = first_kind("request-first") + exchange.second = KeyboardInterrupt("http-exit-second") if http_exit_failure else None + assert_first( + state, + exchange, + caller, + first_kind, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert phase in state.events and len(exchange.exit_args) == 3 + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, ValueError, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +async def test_standalone_nonce_cleanup_keeps_cancellation_and_success_neighbors( + native_call, exchange, caller, owned, cleanup_kind, phase +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-only") if cleanup_kind is not None else None + kind, result = await observed(lambda: state.invoke(caller, owned)) + if cleanup_kind in FATAL_ERRORS: + assert kind == "error" + if owned and cleanup_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is state.secondary + assert state.write_rows.await_count == 0 + else: + assert kind == "value" + assert result == ( + 0 if caller == "prepare" else None if caller == "begin" else exchange.answer + ) + assert state.events == list(CLEANUP_PHASES) + assert state.write_rows.await_count == int(caller == "prepare") + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", FATAL_ERRORS) +async def test_nonce_write_failure_stays_first(native_call, exchange, caller, owned, cleanup_kind): + state = native_call + state.write_first = True + state.secondary = cleanup_kind("nonce-second") + exchange.first = ValueError("write-first") + assert_first( + state, + exchange, + caller, + ValueError, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert exchange.events == [] and state.events == ["postgres"] + + +@pytest.mark.parametrize("caller", CALLERS) +async def test_repeated_cancellation_keeps_first_stop_through_nonce_deletion( + native_call, exchange, caller +): + state = native_call + state.cleanup_block = True + exchange.block = True + task = asyncio.create_task(observed(lambda: state.invoke(caller, False))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(state.cleanup_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + assert state.write_rows.await_count == 0 + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py new file mode 100644 index 0000000000..a8536a27b4 --- /dev/null +++ b/tests/unit/test_cell_place.py @@ -0,0 +1,594 @@ +"""Operator placement keeps the directory winner and bounds its local session.""" + +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import asynccontextmanager +from dataclasses import asdict +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.unit.test_router import _row + +OPERATOR = "lco_" + "7" * 128 + + +@pytest.fixture(autouse=True) +def _code_org_reader(monkeypatch): + monkeypatch.setattr(cell, "_code_orgs", None) + + +class DirectoryConnection: + def __init__(self, winner: str | None = "eu-1") -> None: + self.winner = winner + self.row = _row() + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + self.committed = False + + @asynccontextmanager + async def transaction(self): + yield + self.committed = True + + async def execute(self, sql, *args): + self.calls.append((sql, args)) + + async def fetchrow(self, sql, *args): + self.calls.append((sql, args)) + if "FROM loom_router.cells WHERE" in sql: + return asdict(self.row) + if self.winner is not None: + return {"cell": self.winner, "deployment_id": f"dep-{self.winner}"} + return None + + async def fetchval(self, sql, *args): + self.calls.append((sql, args)) + self.winner = args[1] + return args[0] + + +@pytest.fixture +def connected(monkeypatch): + conn = DirectoryConnection() + + @asynccontextmanager + async def connection(**kw): + yield conn + + monkeypatch.setattr(directory, "_connection", connection) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + return conn + + +def request(client, **over): + return client.post( + "/cells/v1/operator/cells/eu-1/place", + json={ + "org": "acme", + "deployment_id": "dep-eu-1", + "fingerprint": cell.as_value( + {"public_host": "eu-1.example.test", "deployment_id": "dep-eu-1"} + ), + **over, + }, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + +def test_existing_winner_emits_audit_wal_before_returning(connected): + client = TestClient(app.build_app(app.Router())) + result = request(client) + assert result.status_code == 200 + assert result.json() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + audit = [(sql, args) for sql, args in connected.calls if "INSERT INTO loom_router.audit" in sql] + assert len(audit) == 1 and audit[0][1] == ("operator", {"org": "acme", "cell": "eu-1"}) + assert connected.committed + assert not any("UPDATE loom_router.cells SET placed" in sql for sql, _ in connected.calls) + + +@pytest.mark.parametrize( + "posted", + [{"deployment_id": "other"}, {"fingerprint": {}}, {"org": "x" * (cell.ORG_MAX_BYTES + 1)}], +) +def test_existing_winner_refuses_operator_identity_mismatch(connected, posted): + result = request(TestClient(app.build_app(app.Router())), **posted) + assert result.status_code in (400, 409) + assert not connected.committed + assert not any("INSERT" in sql for sql, _ in connected.calls) + + +def test_operator_authentication_precedes_directory_access(connected): + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", json={} + ) + assert result.status_code == 401 and connected.calls == [] + + +def test_live_duplicate_also_emits_audit_wal(connected): + assert asyncio.run( + directory.place("acme", "eu-1", lambda row: "closed", actor="cell:eu-1") + ) == ("eu-1", "dep-eu-1") + assert connected.committed + assert any("INSERT INTO loom_router.audit" in sql for sql, _ in connected.calls) + + +class LocalConnection: + def __init__(self): + self.calls = [] + self.committed = False + self.terminated = False + self.closed = False + self.stall_commit = False + self.stall_close = False + self.fail_insert = False + self.commit_started = asyncio.Event() + + def transaction(self): + conn = self + + class Transaction: + async def start(self): + pass + + async def commit(self): + conn.commit_started.set() + if conn.stall_commit: + await asyncio.Event().wait() + conn.committed = True + + return Transaction() + + async def execute(self, sql, *args): + self.calls.append(sql) + if "INSERT" in sql and self.fail_insert: + raise OSError("local insert failed") + return "INSERT 0 0" + + async def close(self, **kwargs): + self.calls.append("close") + if self.stall_close: + await asyncio.Event().wait() + self.closed = True + + def terminate(self): + self.terminated = True + + def is_closed(self): + return self.closed or self.terminated + + +@pytest.fixture +def local(monkeypatch): + from server.cli import cell_place + from server.core import pg + + conn = LocalConnection() + state = {"first": "eu-1", "state": "enforced", "deployment": "dep-eu-1", "row": None} + + async def read_state(c): + return state["state"], state["first"] + + async def fetchval(sql): + return state["deployment"] + + async def fetchrow(sql): + return state["row"] + + conn.fetchval = fetchval + conn.fetchrow = fetchrow + + async def connect(**kw): + assert kw == {"statement_timeout_ms": settings.pg_statement_timeout_ms} + return conn + + async def initialize(c): + assert c is conn + + async def direct(c, what): + return [] + + async def take_lock(c, what): + conn.calls.append("lock") + + async def held(c): + return True + + async def directory_call(*a): + conn.calls.append("directory") + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(pg, "_check_deployment", initialize) + monkeypatch.setattr(pg, "init_connection", initialize) + monkeypatch.setattr(cell, "read_state_if_present", read_state) + monkeypatch.setattr(cell_place.apg, "direct_refusals", direct) + monkeypatch.setattr(cell_place.apg, "take_lock", take_lock) + monkeypatch.setattr(cell_place.apg, "lock_held", held) + monkeypatch.setattr(cell_place, "_directory", directory_call) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.03) + fields = cell.fingerprint("dep-eu-1") + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + return conn, state + + +async def invoke_place(): + import argparse + + import httpx + + from server.cli import cell_place + + return await cell_place.place( + argparse.Namespace(cell="eu-1", org="acme"), httpx.URL("https://router.test") + ) + + +async def test_local_noop_insert_still_commits_durability_message(local): + from server.core import pg + + conn, state = local + assert await invoke_place() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + assert conn.committed and conn.closed and not conn.terminated + assert conn.calls[0:2] == ["lock", "directory"] + assert conn.calls[-2:] == [pg._DURABLE_MESSAGE, "close"] + + +@pytest.mark.parametrize( + "change", + [ + {"state": "recording"}, + {"state": "partial"}, + {"first": "us-1"}, + {"deployment": "other"}, + {"row": None}, + ], +) +async def test_local_invalid_identity_refuses_before_directory(local, change): + from server.cli import cell_place + + conn, state = local + state.update(change) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert conn.calls == ["lock", "close"] and conn.closed and not conn.terminated + + +@pytest.mark.parametrize("change", [{"state": "recording"}, {"first": "us-1"}, {"row": None}]) +async def test_local_identity_is_read_again_before_insert(local, monkeypatch, change): + from server.cli import cell_place + + conn, state = local + + async def changed(*a): + state.update(change) + + monkeypatch.setattr(cell_place, "_directory", changed) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert not any("INSERT" in sql for sql in conn.calls) + assert conn.closed and not conn.terminated and not conn.committed + + +async def test_local_insert_failure_can_be_retried(local): + conn, state = local + conn.fail_insert = True + with pytest.raises(OSError, match="local insert failed"): + await invoke_place() + assert not conn.committed and conn.closed and not conn.terminated + conn.fail_insert = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_and_cleanup_stalls_are_bounded(local): + conn, state = local + conn.stall_commit = conn.stall_close = True + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert conn.commit_started.is_set() and conn.terminated and not conn.committed + conn.stall_commit = conn.stall_close = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_cancellation_closes_the_session(local): + conn, state = local + conn.stall_commit = True + task = asyncio.create_task(invoke_place()) + await conn.commit_started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert conn.closed and not conn.terminated and not conn.committed + + +async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): + from server.core import pg + + conn, state = local + cleaned = False + + async def stalled(**kw): + nonlocal cleaned + try: + await asyncio.Event().wait() + finally: + cleaned = True + + monkeypatch.setattr(pg, "_connect_dedicated", stalled) + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert cleaned and conn.calls == [] + + +async def test_query_timeout_cannot_enter_an_unbounded_rollback(local): + conn, state = local + + class Transaction: + async def start(self): + pass + + async def commit(self): + pytest.fail("a stalled statement cannot commit") + + async def __aenter__(self): + await self.start() + + async def __aexit__(self, *exc): + await asyncio.Event().wait() + + async def stalled(sql, *args): + await asyncio.Event().wait() + + conn.transaction = Transaction + conn.execute = stalled + task = asyncio.create_task(invoke_place()) + try: + done, _ = await asyncio.wait([task], timeout=0.15) + assert done, "the statement deadline must include ending its transaction" + with pytest.raises(TimeoutError): + await task + assert conn.closed and not conn.terminated + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize( + "status,answer", + [ + (200, {"org": "acme", "cell": "us-1", "deployment_id": "dep-us-1"}), + (200, {"org": "acme", "cell": "eu-1", "deployment_id": "wrong"}), + (200, []), + (200, {"org": "someone-else", "cell": "eu-1", "deployment_id": "dep-eu-1"}), + (503, {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"}), + (504, {"error": "gateway timeout"}), + ], +) +async def test_cli_refuses_unknown_response_or_other_winner(monkeypatch, status, answer): + import httpx + + from server.cli import cell_place + + async def respond(request): + assert request.headers["authorization"] == f"Bearer {settings.router_operator_token}" + return httpx.Response(status, json=answer) + + real = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real(transport=httpx.MockTransport(respond), **kw) + ) + with pytest.raises(cell_place.PlacementRefused): + await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +async def test_repository_only_configured_orgs_are_in_full_cli_fingerprint(local, monkeypatch): + from server.cli import cell_place + from server.code.domain.index import repos + + conn, state = local + monkeypatch.setattr(repos, "operator_orgs", lambda: {"repo-only"}) + fields = cell.fingerprint("dep-eu-1") + fields["configured_orgs"] = sorted(set(fields["configured_orgs"]) | {"repo-only"}) + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + _, value = await cell_place._identity(conn, "eu-1") + assert "repo-only" in value["fields"]["configured_orgs"] + + +def test_new_placement_uses_counted_statement_bound(monkeypatch): + from tests.unit.test_router_attach import _cell_row, _Conn, _Store + + store = _Store(_cell_row(state="serving")) + + @asynccontextmanager + async def connection(**kw): + yield _Conn(store) + + monkeypatch.setattr(directory, "_connection", connection) + asyncio.run( + directory.place("new", "eu-1", lambda row: "", actor="operator", verify=lambda row: "") + ) + assert store.statements == directory.PLACE_STATEMENTS + assert directory.PLACE_MAX_S == directory._bound_s(store.statements) + + +def test_operator_placement_body_bound_refuses_before_directory(connected): + from server.router.limits import MAX_BODY_BYTES + + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", + content=b"x" * (MAX_BODY_BYTES + 1), + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + assert result.status_code == 413 and connected.calls == [] + + +def test_operator_place_bypasses_cell_token_rate_limiter(connected, monkeypatch): + router = app.Router() + monkeypatch.setattr( + router.placements, "admit", lambda name: pytest.fail("operator used cell rate") + ) + assert request(TestClient(app.build_app(router))).status_code == 200 + + +@pytest.mark.parametrize("org", ["", "-", "*", "bad\x00org"]) +def test_operator_place_refuses_malformed_org_before_directory(connected, org): + assert request(TestClient(app.build_app(app.Router())), org=org).status_code == 400 + assert connected.calls == [] + + +def test_cli_cancellation_names_the_idempotent_retry_and_preserves_cancellation( + monkeypatch, capsys +): + import argparse + + import httpx + + from server.cli import cell_place + + cancelled = asyncio.CancelledError("commit response was cancelled") + + async def cancel(*args): + raise cancelled + + monkeypatch.setattr(cell_place, "_args_problem", lambda args: "") + monkeypatch.setattr( + cell_place.cell_cli, "operator_router", lambda args: httpx.URL("https://router.test") + ) + monkeypatch.setattr(cell_place, "place", cancel) + monkeypatch.setattr(cell_place, "run_connected", lambda run, **kw: asyncio.run(run(None))) + with pytest.raises(asyncio.CancelledError) as caught: + cell_place.cmd_place(argparse.Namespace(cell="eu-1", org="acme")) + assert caught.value is cancelled + printed = capsys.readouterr() + assert printed.out == "" + assert "placement may have committed; rerun the same command" in printed.err + + +async def test_local_phases_remain_separate_from_directory_and_exit(local, monkeypatch): + from server.cli import cell_control, cell_place + + observed = [] + actual_call = cell_control.Owner.call + + async def timed(self, work, *, total_s): + observed.append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def directory_call(*args): + # The complete HTTP exchange has its independent directory-derived phase. + await asyncio.sleep(0.06) + + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(cell_place, "_directory", directory_call) + await invoke_place() + assert observed == [0.03, 0.03, cell_place.REQUEST_S, 0.03] + conn, _ = local + assert conn.committed and conn.closed and not conn.terminated + + +@pytest.mark.parametrize("where", ["initialization", "initial", "transaction", "directory"]) +async def test_original_error_survives_failed_required_close(local, monkeypatch, where): + from server.cli import cell_place + from server.core import pg + + original = ValueError("original placement failure") + conn, _ = local + + async def fail(*args): + raise original + + async def bad_close(**kwargs): + raise RuntimeError("close failed") + + conn.close = bad_close + if where == "initialization": + monkeypatch.setattr(pg, "init_connection", fail) + elif where == "initial": + monkeypatch.setattr(cell_place, "_identity", fail) + elif where == "directory": + monkeypatch.setattr(cell_place, "_directory", fail) + else: + conn.execute = fail + with pytest.raises(ValueError) as caught: + await invoke_place() + assert caught.value is original + assert conn.terminated and not conn.committed + + +async def test_success_refuses_when_required_close_fails(local): + conn, _ = local + + async def fail(**kwargs): + raise OSError("required close failed") + + conn.close = fail + with pytest.raises(OSError, match="required close failed"): + await invoke_place() + assert conn.committed and conn.terminated + + +async def test_lost_original_lock_refuses_without_reconnecting(local, monkeypatch): + from server.cli import cell_place + + conn, _ = local + + async def lost(c): + assert c is conn + return False + + monkeypatch.setattr(cell_place.apg, "lock_held", lost) + with pytest.raises(cell_place.PlacementRefused, match="lock was lost"): + await invoke_place() + assert conn.calls == ["lock", "directory", "close"] + assert conn.closed and not conn.committed + + +@pytest.mark.parametrize("inside_reserve", [False, True]) +async def test_late_placement_connection_never_reaches_body(local, monkeypatch, inside_reserve): + from server.core import pg + + conn, _ = local + released, discarded = asyncio.Event(), asyncio.Event() + real_terminate = conn.terminate + real_close = conn.close + + def terminate(): + real_terminate() + discarded.set() + + async def close(**kw): + await real_close(**kw) + discarded.set() + + async def late(**kwargs): + try: + await asyncio.Future() + except asyncio.CancelledError: + if not inside_reserve: + while not released.is_set(): + try: + await released.wait() + except asyncio.CancelledError: + pass + return conn + + conn.terminate, conn.close = terminate, close + monkeypatch.setattr(pg, "_connect_dedicated", late) + with pytest.raises(TimeoutError): + await invoke_place() + released.set() + await asyncio.wait_for(discarded.wait(), 0.2) + assert conn.calls == (["close"] if inside_reserve else []) + assert conn.closed if inside_reserve else conn.terminated + assert not conn.committed diff --git a/tests/unit/test_cell_place_envelopes.py b/tests/unit/test_cell_place_envelopes.py new file mode 100644 index 0000000000..40b89fabda --- /dev/null +++ b/tests/unit/test_cell_place_envelopes.py @@ -0,0 +1,332 @@ +"""Every accepted fingerprint fits both placement envelopes for every valid org.""" + +from __future__ import annotations + +import hashlib +import itertools +import json +from types import SimpleNamespace +from uuid import UUID + +import httpx +import pytest +from starlette.requests import Request + +from server.core import cell +from server.router import app, directory +from server.router.limits import MAX_BODY_BYTES +from server.settings import settings + +DEPLOYMENT = str(UUID(int=1)) +TOKEN = "lco_" + "7" * 128 + + +def fingerprint(): + return cell.as_value( + { + **cell.DEFAULTS, + "deployment_id": DEPLOYMENT, + "public_host": "first.example.test", + "oauth_issuer_host": "first.example.test", + } + ) + + +def envelopes(name, value, org=""): + return ( + {"org": org, "cell": name, "fingerprint": value}, + {"org": org, "deployment_id": DEPLOYMENT, "fingerprint": value}, + ) + + +def attaching(name): + row = SimpleNamespace( + cell=name, + is_first=True, + state="attaching", + mode="recording", + deployment_id=DEPLOYMENT, + current_fp=None, + next_fp=None, + host="first.example.test", + ) + return SimpleNamespace(row=row, other=None, attach_started_at=None, bucket="", bucket_prefix="") + + +def begin(value, position): + return directory.Begin( + DEPLOYMENT, + value if position == "current_fp" else fingerprint(), + value if position == "next_fp" else None, + "a" * 64, + {}, + ) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_reserves_operator_envelope_before_accepting_fingerprint(name, position): + value = fingerprint() + live = envelopes(name, value)[0] + # This fits the previous live-only reservation, but not the operator envelope. + live_size = len(json.dumps(live)) + 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - live_size) + reason = directory.begin_refusal(attaching(name), begin(value, position)) + assert position in reason and str(MAX_BODY_BYTES) in reason + assert "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("size", ["default", "boundary"]) +@pytest.mark.parametrize("org", ["acme", "\x01" * cell.ORG_MAX_BYTES]) +async def test_accepted_fingerprint_fits_both_encodings_and_operator_handler( + monkeypatch, name, position, size, org +): + value = fingerprint() + if size == "boundary": + empty_size = max(len(json.dumps(body)) for body in envelopes(name, value)) + value["fields"]["oidc_client_id"] = "x" * ( + MAX_BODY_BYTES - empty_size - 6 * cell.ORG_MAX_BYTES + ) + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + attempt = begin(value, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + bodies = envelopes(name, value, org) + encoded = [ + (json.dumps(body).encode(), httpx.Request("POST", "http://inert.test", json=body).content) + for body in bodies + ] + assert all(len(raw) <= MAX_BODY_BYTES for pair in encoded for raw in pair) + calls = [] + + async def place(actual_org, actual_name, refusal, *, actor, verify): + calls.append((actual_org, actual_name, actor)) + locked = SimpleNamespace( + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + ) + assert verify(locked) == "" + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + for raw in encoded[1]: + scope = { + "type": "http", + "method": "POST", + "path": f"/cells/v1/operator/cells/{name}/place", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(raw=raw): + return {"type": "http.request", "body": raw, "more_body": False} + + response = await app.Router().operator_place(Request(scope, receive)) + assert response.status_code == 200 + assert json.loads(response.body) == {"org": org, "cell": name, "deployment_id": DEPLOYMENT} + assert calls == [(org, name, "operator")] * 2 + + +OMITTABLE_DEFAULTS = ( + "bearer_issuers", + "bucket", + "bucket_prefix", + "configured_orgs", + "end_user_orgs", + "oidc_issuer", + "scim", +) + + +@pytest.fixture +def native_fingerprint(monkeypatch): + configured = SimpleNamespace( + public_base_url="https://first.example.test", + oauth_issuer="", + oidc_issuer="", + oidc_client_id="", + scim_token="", + document_s3_bucket="", + document_s3_prefix="", + oidc_jit_org="", + self_signup="closed", + self_signup_org="", + code_org="", + tracker_org="", + trusted_issuers=lambda: set(), + end_user_issuers=lambda: {}, + ) + monkeypatch.setattr(cell, "settings", configured) + monkeypatch.setattr(cell, "_code_orgs", None) + + def native(client_id): + configured.oidc_client_id = client_id + # Both native placement callers serialize this complete field set. + return cell.as_value(cell.fingerprint(DEPLOYMENT)) + + return native + + +def omitted_values(): + return ( + subset + for count in range(len(OMITTABLE_DEFAULTS) + 1) + for subset in itertools.combinations(OMITTABLE_DEFAULTS, count) + ) + + +def stored_fingerprint(native, omitted, unknown): + return cell.as_value( + { + **{key: value for key, value in native["fields"].items() if key not in omitted}, + **unknown, + } + ) + + +def accepted_row(name, attempt): + return SimpleNamespace( + cell=name, + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + state="serving", + is_first=True, + mode="enforced", + closed=False, + holds=lambda token: True, + ) + + +async def route_response(monkeypatch, name, attempt, body, operator): + calls = [] + row = accepted_row(name, attempt) + + async def place(org, actual_name, refusal, *, actor, verify=None): + assert actual_name == name and refusal(row) == "" + if verify is not None: + assert verify(row) == "" + calls.append((org, actor)) + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + router = app.Router() + + async def authenticated(request, actual_name): + assert actual_name == name + return "inert-cell-credential" + + monkeypatch.setattr(router, "_authenticated", authenticated) + monkeypatch.setattr(router, "placements", SimpleNamespace(admit=lambda name: True)) + raw = httpx.Request("POST", "http://inert.test", json=body).content + scope = { + "type": "http", + "method": "POST", + "path": "/cells/v1/placements", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(): + return {"type": "http.request", "body": raw, "more_body": False} + + handler = router.operator_place if operator else router.place + response = await handler(Request(scope, receive)) + return response, calls, len(raw) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x"}, {"future": "x" * 512}]) +def test_sparse_admission_reserves_matching_native_callers( + native_fingerprint, name, position, unknown +): + org = "\x01" * cell.ORG_MAX_BYTES + for omitted in omitted_values(): + stored = stored_fingerprint(native_fingerprint(""), omitted, unknown) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + stored["fields"]["oidc_client_id"] = client_id + native = native_fingerprint(client_id) + attempt = begin(stored, position) + row = accepted_row(name, attempt) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(row, native) + reason = directory.begin_refusal(attaching(name), attempt) + if not reason: + # Admission must cover what these callers send, even if storage is sparse. + assert all( + len(httpx.Request("POST", "http://inert.test", json=body).content) <= MAX_BODY_BYTES + for body in envelopes(name, native, org) + ), (omitted, unknown) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +async def test_sparse_boundary_refuses_unusable_native_requests( + monkeypatch, native_fingerprint, name, position, operator +): + stored = stored_fingerprint(native_fingerprint(""), OMITTABLE_DEFAULTS, {}) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + stored["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(stored["fields"]["oidc_client_id"]) + attempt = begin(stored, position) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + body = envelopes(name, native, org)[int(operator)] + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + if org == "acme": + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + else: + assert size > MAX_BODY_BYTES + assert response.status_code == (413 if operator else 400) + assert not calls + reason = directory.begin_refusal(attaching(name), attempt) + assert position in reason and "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x" * 512}]) +async def test_accepted_sparse_boundary_serves_native_requests( + monkeypatch, native_fingerprint, name, position, operator, unknown +): + native = native_fingerprint("") + # Unknown stored fields keep their existing reservation, although native callers + # do not emit them. Reserve that representation with all known defaults present. + reserved = cell.as_value({**native["fields"], **unknown}) + empty = max(len(json.dumps(body)) for body in envelopes(name, reserved)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(client_id) + stored = stored_fingerprint(native, OMITTABLE_DEFAULTS, unknown) + attempt = begin(stored, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + body = envelopes(name, native, org)[int(operator)] + assert len(json.dumps(body).encode()) <= MAX_BODY_BYTES + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + stored["fields"]["future"] = "x" * MAX_BODY_BYTES + assert position in directory.begin_refusal(attaching(name), begin(stored, position)) diff --git a/tests/unit/test_cell_place_http_deadline.py b/tests/unit/test_cell_place_http_deadline.py new file mode 100644 index 0000000000..79d278903a --- /dev/null +++ b/tests/unit/test_cell_place_http_deadline.py @@ -0,0 +1,157 @@ +"""A complete HTTP exchange cannot outlive its directory-derived owner phase.""" + +from __future__ import annotations + +import asyncio +import json +import time + +import httpcore +import httpx +import pytest +from httpcore._backends.mock import AsyncMockBackend, AsyncMockStream + +from server.cli import cell_control, cell_place +from server.settings import settings +from tests.unit import test_cell_place as helpers + +local = helpers.local +invoke_place = helpers.invoke_place + +_DIRECTORY = cell_place._directory + + +@pytest.fixture +def exchange(local, monkeypatch): + conn, _ = local + events = [] + entered = asyncio.Event() + state = {"mode": "response", "reads": [], "owners": []} + real_close = conn.close + real_call = cell_control.Owner.call + actual_client = httpx.AsyncClient + + async def close(**kwargs): + events.append("pg-close") + await asyncio.sleep(0) + await real_close(**kwargs) + events.append("pg-ack") + + async def timed(owner, work, *, total_s): + if total_s == 0.1: + state["owners"].append(owner) + return await real_call(owner, work, total_s=total_s) + + class Stream(AsyncMockStream): + def __init__(self): + super().__init__([]) + self.headers = False + + async def read(self, max_bytes, timeout=None): # noqa: ASYNC109 — httpcore transport API + if not self.headers: + self.headers = True + if state["mode"] == "response": + return b"HTTP/1.1 200 OK\r\nContent-Length: 100000000\r\n\r\n" + status = b"503 Unavailable" if state["mode"] == "refusal" else b"200 OK" + body = json.dumps( + {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"} + if state["mode"] == "refusal" + else {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + ).encode() + # A timely reply may exceed the separate local phase allowance. + await asyncio.sleep(0.04) + return ( + b"HTTP/1.1 " + + status + + b"\r\nContent-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + entered.set() + state["reads"].append(timeout) + try: + async with asyncio.timeout(timeout): + await asyncio.sleep(0.002) + return b" " + except asyncio.CancelledError: + events.append("http-cancel") + raise + + class Backend(AsyncMockBackend): + async def connect_tcp(self, *args, **kwargs): + return Stream() + + class Client(actual_client): + async def __aexit__(self, *args): + if state["mode"] == "client-exit": + entered.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + events.append("http-cancel") + raise + return await super().__aexit__(*args) + + def client(**kwargs): + transport = httpx.AsyncHTTPTransport() + transport._pool = httpcore.AsyncConnectionPool(network_backend=Backend([])) + return Client(transport=transport, **kwargs) + + conn.close = close + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "REQUEST_S", 0.1, raising=False) + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return conn, events, entered, state + + +@pytest.mark.parametrize("blocking", ["response", "client-exit"]) +@pytest.mark.parametrize("stop", ["deadline", "cancel"]) +async def test_whole_exchange_closes_pg_before_http_cancellation(exchange, blocking, stop): + conn, events, entered, state = exchange + state["mode"] = blocking + started = time.monotonic() + task = asyncio.create_task(invoke_place()) + try: + await asyncio.wait_for(entered.wait(), 0.2) + if stop == "cancel": + task.cancel("operator stop") + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "per-read HTTP timeouts cannot bound a continuing response or client cleanup" + with pytest.raises( + TimeoutError if stop == "deadline" else asyncio.CancelledError + ) as caught: + await task + if stop == "cancel": + assert str(caught.value) == "operator stop" + assert time.monotonic() - started < 0.3 + assert conn.closed and not conn.committed and not conn.terminated + assert not any("INSERT" in call for call in conn.calls) + assert events.index("pg-ack") < events.index("http-cancel") + assert events.count("pg-close") == 1 + if stop == "deadline": + (owner,) = state["owners"] + assert owner.close_end == owner.phase_end + if blocking == "response": + assert state["reads"] and all(value == 0.1 for value in state["reads"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("mode", ["success", "refusal"]) +async def test_timely_actual_http_reply_keeps_local_phase_bounds(exchange, mode): + conn, events, _, state = exchange + state["mode"] = mode + if mode == "refusal": + with pytest.raises(cell_place.PlacementRefused, match="standby stalled"): + await invoke_place() + assert not conn.committed + else: + assert (await invoke_place())["deployment_id"] == "dep-eu-1" + assert conn.committed + assert conn.closed and not conn.terminated + assert events.count("pg-close") == 1 diff --git a/tests/unit/test_cell_place_http_first_error.py b/tests/unit/test_cell_place_http_first_error.py new file mode 100644 index 0000000000..b8120702c9 --- /dev/null +++ b/tests/unit/test_cell_place_http_first_error.py @@ -0,0 +1,164 @@ +"""Placement HTTP failures keep their outcome through client and Postgres cleanup.""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from server.cli import cell_place +from tests.unit import test_cell_native_http_first_error as native +from tests.unit import test_cell_place as place_helpers + +local = place_helpers.local +invoke_place = place_helpers.invoke_place +exchange = native.exchange +observed = native.observed +assert_first = native.assert_first +assert_exit_first = native.assert_exit_first +assert_close_failure = native.assert_close_failure +FIRST_ERRORS = native.FIRST_ERRORS +EXIT_ERRORS = native.EXIT_ERRORS +_BOUNDARIES = ("placement",) +_DIRECTORY = cell_place._directory + + +async def invoke(boundary): + return await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + assert result is None + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_keeps_http_first_failure_and_closes_pg_once( + local, exchange, monkeypatch, first_kind, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") + kind, result = await observed(invoke_place) + assert kind == "error" + if first_kind is asyncio.CancelledError: + # A cancelled child Task reconstructs this exception when Owner reads it. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_required_http_close_failure_prevents_insert( + local, exchange, monkeypatch, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.second = second_kind("required-exit") + kind, result = await observed(invoke_place) + assert kind == "error" + if second_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.second + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +async def test_owned_placement_successful_http_neighbor_commits(local, exchange, monkeypatch): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + assert await invoke_place() == exchange.answer + assert conn.calls.count("close") == 1 + assert conn.closed and conn.committed and not conn.terminated diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py index 2cce1d25cf..1b8cd967e7 100644 --- a/tests/unit/test_cell_place_owner.py +++ b/tests/unit/test_cell_place_owner.py @@ -163,6 +163,80 @@ def is_closed(self): await asyncio.gather(task, return_exceptions=True) +@pytest.mark.parametrize("stage", ["opening", "initial", "directory", "transaction", "close"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +async def test_placement_fatal_stays_inside_owner_until_public_close(stage, kind): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse, asyncio, sys + import httpx + from server.cli import cell_place + from server.core import pg + stage, kind = sys.argv[1:] + original = (SystemExit if kind == 'SystemExit' else KeyboardInterrupt)('original') + events = [] + class Transaction: + async def start(self): pass + async def commit(self): events.append('commit') + class Connection: + def transaction(self): return Transaction() + async def execute(self, *args): + if stage == 'transaction': raise original + async def close(self, **kw): + events.append('close') + if stage == 'close': raise original + def terminate(self): events.append('abort') + conn = Connection() + async def connect(**kw): return conn + async def initialize(c): + assert c is conn + if stage == 'opening': raise original + async def noop(*args): pass + async def direct(c, what): return [] + async def identity(c, name): + assert c is conn + if stage == 'initial': raise original + return 'dep', {} + async def directory(*args): + if stage == 'directory': raise original + async def held(c): return True + async def main(): + pg._connect_dedicated = connect + pg._check_deployment = initialize + pg.init_connection = noop + cell_place.apg.direct_refusals = direct + cell_place.apg.take_lock = noop + cell_place.apg.lock_held = held + cell_place._identity = identity + cell_place._directory = directory + try: + await cell_place.place(argparse.Namespace(cell='first', org='acme'), httpx.URL('http://router')) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 1, events + await asyncio.sleep(0) + print('original-after-public-close') + return + raise AssertionError('fatal became success') + asyncio.run(main()) + """ + ) + result = await asyncio.to_thread( + subprocess.run, + [sys.executable, "-c", program, stage, kind], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-public-close" in result.stdout + + async def test_repeated_signals_cannot_extend_an_expired_public_close(): closing = asyncio.Event() conn = FakeConn() diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 84bd4db79d..c240249eaa 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -84,7 +84,9 @@ async def first_cell_named(self, name: str) -> directory.Cell | None: row = self.cells.get(name) return row if row is not None and row.is_first else None - async def place(self, org: str, name: str, refusal: Any) -> tuple[str, str]: + async def place( + self, org: str, name: str, refusal: Any, *, actor: str, verify: Any = None + ) -> tuple[str, str]: self._check() if org not in self.placements: row = self.cells[name] # the row as the placement locks it diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 821272bcdf..4b0ec1f35c 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -970,7 +970,7 @@ def observed(name: str = "eu-1") -> directory.Control: def test_a_live_placement_reads_the_winner_again_under_the_row_lock(store: _Store) -> None: """A batch holds the cell's row lock, not the org's: an org it placed between the live placement's first read and its row lock is answered placed, never judged on the quota, - counted or audited twice.""" + counted twice. Its call still writes the durability audit.""" store.cells["eu-1"].update(state="serving", placements_max=0) reads = 0 real = _Conn.fetchrow @@ -985,9 +985,10 @@ async def _fetchrow(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchrow", _fetchrow) - winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) + winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) assert winner == ("eu-1", "dep-1") and reads == 2 - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] def test_a_live_insert_that_loses_to_the_primary_key_counts_nothing(store: _Store) -> None: @@ -1001,8 +1002,12 @@ async def _fetchval(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchval", _fetchval) - assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) == ("eu-1", "dep-1") - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) == ( + "eu-1", + "dep-1", + ) + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] # The attach lock and the troubleshooting page. @@ -1451,3 +1456,28 @@ def test_native_compact_batch_with_control_envelope_fits_public_handler(client, assert response.status_code == 200, response.text assert response.json()["placed"] == len(batch) assert set(store.placements) == set(batch) + + +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_refuses_operator_envelope_overflow_before_probe_or_writes( + client: TestClient, store: _Store, listener: _Listener, position: str +) -> None: + deployment = str(UUID(int=1)) + store.cells["eu-1"]["deployment_id"] = deployment + current = cell.as_value({**FIELDS, "deployment_id": deployment}) + grown = cell.as_value({**FIELDS, "deployment_id": deployment, "oidc_client_id": ""}) + live = {"org": "", "cell": "eu-1", "fingerprint": grown} + available = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES + grown["fields"]["oidc_client_id"] = "x" * available + body = ( + _begin(deployment_id=deployment, current_fp=current, **{position: grown}) + if position == "next_fp" + else _begin(deployment_id=deployment, current_fp=grown) + ) + response = _call(client, "POST", "attach/begin", body) + assert response.status_code == 409 + assert position in response.json()["error"] + assert "oidc_client_id" in response.json()["error"] + assert store.statements == directory.ATTACH_READ_STATEMENTS + assert listener.asked == [] and store.audit == [] + assert store.cells["eu-1"]["attach_started_at"] is None