From bc35583b03beedc9e53baeecd887366995473036 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 04:34:38 +0000 Subject: [PATCH 01/25] loom cell attach turns a prepared first cell into a serving one; prepare takes the same lock The attach command holds the attach lock on a direct, marked session of the cell's Postgres for its run and resumes by the directory row and the cell's mode. It refuses before any write on the stores' identity, the session, the roles, settings and subscriptions that would let an unchecked writer through, the database encoding, durability, its own grants, the ClickHouse writers, the table list and the processes' heartbeats; posts attach/begin; creates the triggers under the migration lock; in recording scans and writes the tenancy rows, refusing an org past ORG_MAX_BYTES; commits enforced, proves it durable, waits out every transaction that could have read recording, rescans, places every org in batches and marks the cell serving. A refusal after the mode commit leaves the cell enforced and attaching, and a rerun resumes there. Prepare refuses a session behind a pooler or a database that is not UTF8, and takes the attach lock. Doctor runs attach's Postgres reads again and reads stray orgs with --stray-orgs. instances.read_recent keeps stopped rows inside the widened window, and PRUNE_AFTER_S names the sweep's hour. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell.py | 4 +- server/cli/cell_attach.py | 756 +++++++++++++++++++ server/cli/cell_attach_beats.py | 324 ++++++++ server/cli/cell_attach_pg.py | 690 +++++++++++++++++ server/cli/cell_prepare.py | 42 +- server/cli/cell_scan.py | 50 +- server/cli/doctor.py | 47 +- server/cli/doctor_cell.py | 166 +++- server/core/cell_triggers.py | 139 ++-- server/core/instances.py | 57 +- tests/integration/test_cell_attach.py | 635 ++++++++++++++++ tests/integration/test_cell_scan.py | 2 +- tests/unit/test_cell_attach.py | 682 +++++++++++++++++ tests/unit/test_cell_org_triggers.py | 2 +- tests/unit/test_cell_prepare.py | 3 + tests/unit/test_doctor_client_checks.py | 2 +- tests/unit/test_doctor_client_client_only.py | 2 +- 17 files changed, 3529 insertions(+), 74 deletions(-) create mode 100644 server/cli/cell_attach.py create mode 100644 server/cli/cell_attach_beats.py create mode 100644 server/cli/cell_attach_pg.py create mode 100644 tests/integration/test_cell_attach.py create mode 100644 tests/unit/test_cell_attach.py diff --git a/server/cli/cell.py b/server/cli/cell.py index 9cdec5c894..65f18990d8 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,7 +1,7 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is -`server/cli/cell_prepare.py`. +`server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; `credential --cell --next | --finish | --revert` rotates a cell's. @@ -459,6 +459,8 @@ def _add_cell_parser(sub: Any) -> None: remove.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") remove.set_defaults(func=_cmd_remove) + from server.cli.cell_attach import add_attach_parser from server.cli.cell_prepare import add_prepare_parser add_prepare_parser(csub) + add_attach_parser(csub) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py new file mode 100644 index 0000000000..368f74ebf8 --- /dev/null +++ b/server/cli/cell_attach.py @@ -0,0 +1,756 @@ +"""`loom cell attach`: turn a prepared first cell into a `serving` one +(docs/architecture/cells.md, "Attaching a cell"; the runbook is +docs/operations/cells.md). + +A run holds the attach lock on a direct session of the cell's Postgres for its life, and +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: + +1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +2. the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, + the table list, durability, attach's own grants; +3. `attach/begin` (the router probes the stores and writes the fingerprints); +4. the mode: `recording`, or `enforced` on a resume; +5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; +7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode + only rises, so every later refusal leaves the cell `enforced` and `attaching`; +8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction + that could have read `recording`, the rescan, the tenancy rows and their placements + (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals +and the scan and prints what would be placed, writing nothing. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from collections.abc import Callable, Iterable +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli import cell_scan +from server.cli._ch import writes +from server.cli.cell_ch import Refusal, Refused, read_topology +from server.core import cell, cell_triggers +from server.settings import settings + +_WAIT_S = 600 # not a setting: a flag's default +_SCAN_BUDGET_S = 7200 # not a setting: a flag's default +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on the lock session's catalog reads +_HEALTH_TIMEOUT = httpx.Timeout(10.0, connect=5.0) # not a setting: one health read + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return f"{args.cell!r} is no cell name (1 to 32 of a-z, 0-9, -)" + if args.cell != settings.cell: + return f"LOOM_CELL is {settings.cell!r}: attach runs with LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return ( + "a cell keeps its state in Postgres: set LOOM_STORAGE_AUTHORITY=postgres and " + "LOOM_PG_DSN" + ) + if args.wait < 0 or args.scan_budget <= 0: + return "--wait is 0 or more seconds, --scan-budget more than 0" + return "" + + +class Router: + """The router's operator API for one cell.""" + + def __init__( + self, url: httpx.URL, name: str, transport: httpx.AsyncBaseTransport | None + ) -> None: + self.url, self.name, self.transport = url, name, transport + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + timeout=cell_cli._OPERATOR_TIMEOUT, + trust_env=self.url.scheme == "https", + transport=self.transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + """The router's 200 answer naming the cell; Refused with its refusal otherwise.""" + url = cell_cli.at(self.url, f"/cells/v1/operator/cells/{self.name}" + path) + try: + async with self._client() as client: + r = await (client.get(url) if body is None else client.post(url, json=body)) + except httpx.HTTPError as e: + raise _refused("router", f"no answer ({type(e).__name__}): rerun attach") from None + answer = cell_cli.router_answer(r, {"cell": self.name}, cell_cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused("router", cell_cli.not_the_router(r)) + if r.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("router", f"a router replica is not upgraded: it does not answer {url}") + if r.status_code != 200: + raise _refused( + "router", f"refused {path or 'the read'} ({r.status_code}): {answer['error']}" + ) + return answer + + async def health(self, host: str) -> str: + """The deployment id `https:///v1/health/deployment` answers ('' when none).""" + try: + async with httpx.AsyncClient( + timeout=_HEALTH_TIMEOUT, transport=self.transport + ) as client: + r = await client.get(f"https://{host}/v1/health/deployment") + answer = r.json() + except (httpx.HTTPError, ValueError): + return "" + return str(answer.get("deployment_id") or "") if isinstance(answer, dict) else "" + + +def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: + """Step 2's refusal of every org past `cell.ORG_MAX_BYTES`, before any tenancy row.""" + return [ + Refusal( + f"org of {len(org.encode())} bytes ({org[:64]}…)", + f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " + "remove it first (docs/operations/cells.md, the over-long org procedure)", + ) + for org, sources in sorted(found.too_long.items()) + ] + + +def _refused(subject: str, reason: str) -> Refused: + return Refused([Refusal(subject, reason)]) + + +def _encoded_size(body: Any) -> int: + return len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + + +def batches(orgs: Iterable[str], deployment_id: str) -> list[list[str]]: + """`orgs` in `attach/place` batches: at most ATTACH_BATCH each, and an encoded body + within the router's body bound.""" + from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES + + out: list[list[str]] = [] + batch: list[str] = [] + for org in sorted(orgs): + if batch and ( + len(batch) >= ATTACH_BATCH + or _encoded_size({"deployment_id": deployment_id, "orgs": [*batch, org]}) + > MAX_BODY_BYTES + ): + out.append(batch) + batch = [] + batch.append(org) + if batch: + out.append(batch) + return out + + +class Run: + """One attach run: its sessions, what it read, and what it prints.""" + + def __init__( + self, + ch: Any, + args: argparse.Namespace, + router: Router, + log: Callable[[str], None], + ) -> None: + self.ch, self.args, self.router, self.log = ch, args, router, log + self.name = args.cell + self.wait = apg.Wait(args.wait) + self.lock_conn: Any = None + self._lock_cm: Any = None + self.row: dict[str, Any] = {} + self.mode = "" + self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) + self.keys_fp = "" + self.report: dict[str, Any] = {"cell": self.name} + + # -- the lock session -------------------------------------------------------------- + + async def open_lock(self) -> None: + from server.core import pg + + self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + self.lock_conn = await self._lock_cm.__aenter__() + if refusals := await apg.direct_refusals(self.lock_conn, "attach"): + raise Refused(refusals) + await apg.take_lock(self.lock_conn, "attach") + + async def close_lock(self) -> None: + if self._lock_cm is not None: + await self._lock_cm.__aexit__(None, None, None) + self._lock_cm = self.lock_conn = None + + async def confirm_lock(self) -> None: + """The lock is held still, or taken again (refusing when another run holds it now), + and the cell's state read again: a cleanup, a prepare rerun or another attach could + have run in the gap.""" + try: + if await apg.lock_held(self.lock_conn): + return + except Exception as e: # noqa: BLE001 — a lost session is taken again below + self.log(f"the lock session was lost ({type(e).__name__}); taking the lock again") + else: + self.log("the attach lock was lost; taking it again") + try: + await self.close_lock() + except Exception: # noqa: BLE001 — the old session is gone either way + self._lock_cm = self.lock_conn = None + await self.open_lock() + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name) or (self.mode == "enforced" and state != "enforced"): + raise _refused( + "cell state", + f"changed while the lock was lost (now {state}, first row {first}): rerun attach", + ) + row = await self.router.call("") + if ( + row.get("deployment_id") != self.row.get("deployment_id") + or row.get("state") != "attaching" + ): + raise _refused("directory row", "changed while the lock was lost: rerun attach") + + # -- the refusals ------------------------------------------------------------------ + + async def read_own_row(self) -> None: + own = await self.lock_conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + if own is None: + raise _refused("postgres loom.cell_self", "absent: run loom cell prepare first") + self.own_fp = (cell.json_object(own["current_fp"]), cell.json_object(own["next_fp"])) + + def judge_beats(self, stopped_unset: str) -> Callable[[list[beats.Beat]], beats.Verdict]: + def _judge(found: list[beats.Beat]) -> beats.Verdict: + if stopped_unset == "ignore": + found = [b for b in found if b.live or b.cell == self.name] + return beats.judge( + found, self.name, self.own_fp, "wait" if stopped_unset == "wait" else "refuse" + ) + + return _judge + + async def check_beats(self, stopped_unset: str) -> beats.Verdict: + verdict = await beats.settle( + lambda: beats.read_beats(self.ch), self.judge_beats(stopped_unset), self.wait + ) + (self.keys_fp,) = verdict.keys_fps + return verdict + + async def stores_refusals(self) -> list[Refusal]: + from server.core import pg_profile + + out = [] + row = self.row + if not row.get("is_first") or row.get("state") != "attaching": + out.append( + Refusal( + "directory row", + f"is {'the first' if row.get('is_first') else 'a later'} cell, " + f"{row.get('state')}: this attach is the first cell's, while attaching", + ) + ) + if row.get("other"): + out.append( + Refusal( + "directory row", + f"another cell, {row['other']}, is registered: remove it first " + "(loom cell remove)", + ) + ) + expected = str(row.get("deployment_id") or "") + recorded_pg = await self.lock_conn.fetchval("SELECT deployment_id FROM loom.authority") + if recorded_pg != expected: + out.append( + Refusal( + "postgres", f"records deployment id {recorded_pg!r}, the directory {expected!r}" + ) + ) + recorded = await pg_profile.recorded_authority(self.ch) + if recorded is None or str(recorded["deployment_id"]) != expected: + out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name): + out.append( + Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") + ) + if state not in ("recording", "enforced"): + out.append(Refusal("postgres cell state", f"reads {state}")) + answered = await self.router.health(str(row.get("host") or "")) + if answered != expected: + out.append( + Refusal( + f"https://{row.get('host')}/v1/health/deployment", + f"answers deployment id {answered!r}, not {expected!r}", + ) + ) + return out + + async def durability_refusals(self) -> list[Refusal]: + from server.core import pg, pg_profile + + out = [] + verdict = pg_profile.durability_verdict("synchronous", *await pg_profile.standby_state()) + if verdict.refusal: + out.append(Refusal("postgres durability", verdict.refusal)) + try: + await pg.check_durable_message() + except Exception as e: # noqa: BLE001 — a role that may not write it refuses + out.append(Refusal("postgres durability", f"pg_logical_emit_message refused: {e}")) + if (now := await pg_profile.standby_now()) != "synchronous": + out.append( + Refusal("postgres standbys", f"commits are acknowledged as {now}, not synchronous") + ) + return out + + async def ch_limit_refusals(self) -> list[Refusal]: + """A profile or quota that caps the rows or bytes attach's user reads would end a + scan early.""" + row = await self.ch.query_one( + "SELECT getSetting('max_rows_to_read') AS r, getSetting('max_bytes_to_read') AS b" + ) + out = [] + if row and (int(row["r"]) or int(row["b"])): + out.append( + Refusal("clickhouse user", "a settings profile caps the rows or bytes it reads") + ) + quota = await self.ch.query_all( + "SELECT quota_name FROM system.quota_usage " + "WHERE max_read_rows > 0 OR max_read_bytes > 0" + ) + out += [ + Refusal(f"clickhouse quota {q['quota_name']}", "caps the rows or bytes attach reads") + for q in quota + ] + return out + + async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: + from server.cli import cell_writers + + topology = await read_topology(self.ch) + if refusals := topology.refusals(): + return refusals, topology, {} + names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + allowed = cell_writers.Allowed( + users=frozenset(self.args.ch_allow_user), + user_directories=frozenset(self.args.ch_allow_user_directory), + objects=frozenset(self.args.ch_allow_object), + replicas=frozenset(self.args.ch_allow_replica), + ) + return await cell_writers.check_writers(self.ch, topology, names, allowed), topology, names + + async def catalog_refusals(self) -> list[Refusal]: + """The Postgres checks that run again before the mode commit and in step 4.""" + holders = await apg.privilege_holders(self.lock_conn) + self.report["own_roles"] = holders.own + out = apg.privilege_refusals(holders, self.args.pg_allow_role) + out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) + return out + + async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: + """Every refusal before any write, named together.""" + out = await self.stores_refusals() + out += await apg.encoding_refusals(self.lock_conn) + out += await self.catalog_refusals() + out += await self.durability_refusals() + rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) + tables_pg = [f"loom.{r['relname']}" for r in rows] + out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await self.ch_limit_refusals() + if out: + raise Refused(out) + state, _ = await apg.read_mode(self.lock_conn) + # A stopped process with LOOM_CELL unset refuses while the cell can still be + # backed out; on a resume in `enforced` step 4 waits for it to age out. + await self.check_beats("refuse" if state == "recording" else "ignore") + writers, topology, names = await self.writer_refusals() + out += writers + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + out += listed.refusals + if out: + raise Refused(out) + return topology, names, listed + + # -- begin ------------------------------------------------------------------------- + + async def begin(self) -> None: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + from server.router.limits import ATTACH_BEGIN_MAX_BYTES + + current, nxt = self.own_fp + nonces = _new_nonces() + body = { + "deployment_id": self.row["deployment_id"], + "current_fp": current, + "next_fp": nxt, + "keys_fp": self.keys_fp, + "ch_allow_users": sorted(self.args.ch_allow_user), + "ch_allow_user_directories": sorted(self.args.ch_allow_user_directory), + "ch_allow_objects": sorted(self.args.ch_allow_object), + "ch_allow_replicas": sorted(self.args.ch_allow_replica), + "pg_allow_roles": sorted(self.args.pg_allow_role), + "nonces": nonces, + } + if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: + raise _refused( + "attach/begin", + f"its body is {size} bytes, past the router's {ATTACH_BEGIN_MAX_BYTES}: the " + "fingerprints or the allow lists are too large", + ) + try: + await _write_nonces(self.ch, nonces) + self.row = {**self.row, **await self.router.call("/attach/begin", body)} + finally: + await _delete_nonces(self.ch, nonces) + + # -- the scans --------------------------------------------------------------------- + + async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: + budget = cell_scan.Budget(self.args.scan_budget) + current, nxt = self.own_fp + return await cell_scan.scan( + self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) + ) + + def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: + return { + "malformed": {k: v[:20] for k, v in sorted(found.malformed.items())}, + "too_long": { + f"{o[:64]}… ({len(o.encode())} bytes)": sorted(s) for o, s in found.too_long.items() + }, + } + + async def triggers(self) -> None: + if refusals := await apg.ensure_triggers(self.lock_conn, self.wait, self.log): + raise Refused(refusals) + + # -- step 3 ------------------------------------------------------------------------ + + async def ready_for_commit(self) -> bool: + """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, + a booting process skips the try.""" + verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) + if verdict.refusals: + raise Refused(verdict.refusals) + return not verdict.booting + + async def commit_mode(self) -> None: + from server.core import pg + + await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + try: + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.commit_enforced( + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + ) + except Refused: + raise + except Exception as e: # noqa: BLE001 — the durable read says whether it landed + self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.end_mode_transactions(conn, self.confirm_lock) + + async def durable(self) -> int: + """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's + xmax, above the read's own xid.""" + from server.core import pg_profile + + while True: + try: + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read() + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") + break + except Refused: + raise + except Exception as e: + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused("postgres durable read", "did not commit within --wait") from e + if read.first != (True, self.name) or read.mode != "enforced": + raise _refused( + "postgres cell state", + f"the durable read saw mode {read.mode!r} and first row {read.first}: the mode " + "commit did not land; rerun attach", + ) + self.mode = "enforced" + xmax = await apg.snapshot_xmax(self.lock_conn) + if xmax <= read.xid: + raise _refused("postgres snapshot", f"its xmax {xmax} is not above {read.xid}") + return xmax + + # -- step 4 ------------------------------------------------------------------------ + + async def cheap_checks(self, stopped_unset: str) -> None: + from server.core import pg_profile + + out, _, _ = await self.writer_refusals() + out += [ + Refusal("postgres triggers", d.text) + for d in await cell_triggers.missing(self.lock_conn) + ] + out += await self.catalog_refusals() + if out: + raise Refused(out) + await self.check_beats(stopped_unset) + if (now := await pg_profile.standby_now()) != "synchronous": + raise _refused("postgres standbys", f"commits are acknowledged as {now}") + + async def final_checks(self, watch: beats.Watch) -> None: + from server.core import instances + + floor = apg.Wait(max(self.wait.left(), instances.boot_allowance_s())) + + async def _read() -> list[beats.Beat]: + found = await watch.read_once() + if refusals := watch.refusals(): + raise Refused(refusals) + # A cell-correct process whose cell row alone was swept. + return [b for b in found if not (b.cell is None and b.instance_id in watch.cell_ids)] + + verdict = await beats.settle( + _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() + ) + if verdict.keys_fps != {self.keys_fp}: + raise _refused( + "processes", + "the root-secret fingerprint changed during attach (a root-secret rotation): " + "rerun attach", + ) + + async def step4(self, xmax: int, topology: Any, names: Any) -> None: + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.wait_transactions(conn, xmax, self.wait) + await self.cheap_checks("wait") + since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) + watch = beats.Watch(self.ch, since, self.name) + if refusals := watch.refusals_up_front(): + raise Refused(refusals) + task = asyncio.create_task(watch.run()) + try: + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + found = await self.scan(topology, listed) + orgs = set(found.orgs) + self.report["tenancy_rows_written"] = await apg.write_tenancy(orgs) + placed = 0 + for batch in batches(orgs, self.row["deployment_id"]): + await self.confirm_lock() + answer = await self.router.call( + "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} + ) + placed += int(answer.get("placed") or 0) + self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) + writers, _, _ = await self.writer_refusals() + if writers: + raise Refused(writers) + finally: + task.cancel() + await self.final_checks(watch) + row = await self.router.call("") + current, nxt = self.own_fp + if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): + raise _refused( + "postgres loom.cell_self", + "differs from the directory's fingerprints (a loom cell update between them?)", + ) + state, _ = await apg.read_mode(self.lock_conn) + if state != "enforced": + raise _refused("postgres cell state", f"reads {state}, not enforced") + + async def serving(self) -> dict[str, Any]: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + + nonces = _new_nonces() + try: + await _write_nonces(self.ch, nonces) + return await self.router.call( + "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} + ) + finally: + await _delete_nonces(self.ch, nonces) + + # -- the run ----------------------------------------------------------------------- + + async def done(self) -> bool: + """On a `serving` row: True once the cell reads `enforced`; Refused otherwise.""" + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + state, _ = await apg.read_mode(conn) + if state != "enforced": + raise _refused( + "postgres cell state", + f"the directory says serving and the cell reads {state}: run loom cell restore", + ) + return True + + async def run(self) -> int: + self.row = await self.router.call("") + if self.row.get("state") == "serving": + await self.done() + self.report.update(state="serving", done=True) + return 0 + await self.open_lock() + try: + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") + await self.triggers() + await self.precommit(found) + await self.commit_mode() + else: + await self.triggers() + await self.router.call( + "/attach/enforced", {"deployment_id": self.row["deployment_id"]} + ) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 + finally: + await self.close_lock() + + async def precommit(self, found: cell_scan.ScanResult) -> None: + from server.core import pg_profile + + out = await self.catalog_refusals() + if (now := await pg_profile.standby_now()) != "synchronous": + out.append(Refusal("postgres standbys", f"commits are acknowledged as {now}")) + if found.elapsed_s > self.args.scan_budget / 2: + out.append( + Refusal( + "--scan-budget", + f"step 2 took {found.elapsed_s:.0f} s, more than half of " + f"{self.args.scan_budget} s, and step 4 reads the same tables again: " + "raise it", + ) + ) + if out: + raise Refused(out) + verdict = await self.check_beats("refuse") + if refusals := beats.final_refusals(verdict): + raise Refused(refusals) + + async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: + found = await self.scan(topology, listed) + per_source: dict[str, list[str]] = {} + for org, sources in found.orgs.items(): + for s in sources: + per_source.setdefault(s, []).append(org) + self.report.update( + dry_run=True, + scan_seconds=round(found.elapsed_s, 1), + half_budget_ok=found.elapsed_s <= self.args.scan_budget / 2, + orgs=len(found.orgs), + per_source={ + s: {"count": len(o), "sample": sorted(o)[:5]} for s, o in sorted(per_source.items()) + }, + skipped=self._named(found), + ) + return 0 + + +async def attach( + ch: Any, + args: argparse.Namespace, + url: httpx.URL, + transport: httpx.AsyncBaseTransport | None = None, +) -> int: + def log(text: str) -> None: + print(f"loom cell attach: {text}", file=sys.stderr) + + run = Run(ch, args, Router(url, args.cell, transport), log) + started = time.monotonic() + try: + code = await run.run() + except Refused as e: + for r in e.refusals: + log(f"refusing: {r}") + mode = ( + "enforced, attaching: rerun attach to resume" + if run.mode == "enforced" + else ("unchanged" if not run.mode else f"{run.mode}") + ) + log(f"the cell stays {mode}") + return 1 + run.report["seconds"] = round(time.monotonic() - started, 1) + print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) + return code + + +def cmd_attach(args: argparse.Namespace) -> int: + if problem := _args_problem(args): + print(f"loom cell attach: {problem}", file=sys.stderr) + return 2 + url = cell_cli.operator_router(args) + if isinstance(url, str): + print(f"loom cell attach: {url}", file=sys.stderr) + return 2 + from server.cli._ch import run_connected + from server.core.ch import statement_timeout + + async def _run(ch: Any) -> int: + try: + return await attach(ch, args, url) + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + print( + f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", + file=sys.stderr, + ) + return 1 + + # Every ClickHouse statement, the connect included, may run as long as a scan may. + with statement_timeout(args.scan_budget): + return run_connected(_run, org_free=writes("attaches the cell; places the orgs it holds")) + + +def add_attach_parser(csub: Any) -> None: + p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Run the refusals and the scan and print what would be placed; write nothing " + "(the router's own begin refusals do not run)", + ) + for flag, what in ( + ("--ch-allow-user", "A ClickHouse user that may write the configured databases"), + ("--ch-allow-user-directory", "A ClickHouse user directory the check cannot read"), + ("--ch-allow-object", "A ClickHouse object (db.name) that writes the databases"), + ("--ch-allow-replica", "A replica outside the cluster (db.table:replica)"), + ("--pg-allow-role", "A Postgres role that may make an org held (the servers' role)"), + ): + p.add_argument(flag, action="append", default=[], help=what) + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_attach) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py new file mode 100644 index 0000000000..a2fc37701d --- /dev/null +++ b/server/cli/cell_attach_beats.py @@ -0,0 +1,324 @@ +"""The first attach's judgment of the cell's processes, from their heartbeats +(docs/architecture/cells.md, "Attaching a cell" and "The heartbeat's keys"). + +Every check reads both heartbeat tables across every deployment sharing the ClickHouse, as +prepare does, stopped rows included (`instances.read_recent`), and judges in one order: + +- `cell` first: a live process whose `LOOM_CELL` is unset or another cell refuses at once; + a stopped one, or a row with no cell row that is no longer live, may still be writing + for the widened window after its stop beat, and is refused, or waited on until it ages + out, as the caller's `stopped_unset` says; +- then the keys a current build reports (`serves`, `marked`, `router`): a cell row + without them refuses, even while booting; +- then booting: a live instance with no cell row yet, no tenancy mode yet, `marked` null, + or `router` null where `serves` is true, is waited on; `marked` false, or `router` false + where `serves` is true, refuses, naming the process and the router's cause; +- and the fingerprint: each live process's matches `loom.cell_self`'s current or next. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +from server.cli.cell_attach_pg import Wait +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, instances + +StoppedUnset = Literal["refuse", "wait"] + + +@dataclass(frozen=True) +class Beat: + """One process: its instance row (`live`: inside the live window and not stopped) and + its cell row (`cell` None when it wrote none), the fingerprint JSON decoded.""" + + instance_id: str + live: bool + stopped: bool + keys_fp: str + last_beat: str + cell: str | None = None + mode: str = "" + value: dict[str, Any] | None = None + + +async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: + rows = await instances.read_recent(ch, since=since) + ids = [str(r["instance_id"]) for r in rows] + cells = await ch.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": ids}, + ) + by_id = {str(r["instance_id"]): r for r in cells} + out = [] + for r in rows: + found = by_id.get(str(r["instance_id"])) + beat = Beat( + instance_id=str(r["instance_id"]), + live=bool(r["live"]), + stopped=bool(r["stopped"]), + keys_fp=str(r["keys_fp"]), + last_beat=str(r["last_beat"]), + cell=None if found is None else str(found["cell"]), + mode="" if found is None else str(found["tenancy_mode"]), + value=None if found is None else _decoded(str(found["fingerprint"])), + ) + out.append(beat) + return out + + +def _decoded(text: str) -> dict[str, Any] | None: + try: + value = json.loads(text) + except ValueError: + return None + return value if isinstance(value, dict) else None + + +@dataclass +class Verdict: + """What one read found: the refusals, the processes waited on (booting, or a stopped + process with `LOOM_CELL` unset aging out), the live processes judged, their `keys_fp` + values, and whether one reports SCIM on.""" + + refusals: list[Refusal] = field(default_factory=list) + booting: list[str] = field(default_factory=list) + aging: list[str] = field(default_factory=list) + live: int = 0 + keys_fps: set[str] = field(default_factory=set) + scim: bool = False + + +def judge( + beats: Iterable[Beat], + name: str, + accepted: Iterable[Mapping[str, Any] | None], + stopped_unset: StoppedUnset, +) -> Verdict: + """The verdict on `beats` for the cell `name`, whose processes run `accepted` (its + `loom.cell_self` current and next).""" + values = [v for v in accepted if v is not None] + out = Verdict() + for b in beats: + who = f"process {b.instance_id}" + if b.cell is None: + if b.live: + out.booting.append(b.instance_id) + else: + _stopped(out, b, "stopped before reporting its LOOM_CELL", stopped_unset) + continue + if b.cell != name: + runs = f"runs with LOOM_CELL={b.cell}" if b.cell else "runs with LOOM_CELL unset" + if b.live: + out.refusals.append(Refusal(who, f"{runs}: set LOOM_CELL={name} on it")) + else: + _stopped(out, b, f"{runs} and stopped", stopped_unset) + continue + if not b.live: + continue + value = b.value or {} + if not all(k in value for k in ("serves", "marked", "router")): + out.refusals.append( + Refusal(who, "reports no serves, marked or router key: upgrade it to this release") + ) + continue + if value["marked"] is False: + out.refusals.append( + Refusal( + who, + "its Postgres sessions lose the tenancy marker (a pooler?): point " + "LOOM_PG_DSN straight at Postgres", + ) + ) + continue + serves = value["serves"] is True + if serves and value["router"] is False: + why = value.get("router_why") or "no cause reported" + out.refusals.append(Refusal(who, f"cannot read its router: {why}")) + continue + if value["marked"] is None or (serves and value["router"] is None) or not b.mode: + out.booting.append(b.instance_id) + continue + fields = value.get("fields") + if not isinstance(fields, dict) or not any(cell.matches(fields, v) for v in values): + out.refusals.append( + Refusal(who, "runs neither loom.cell_self's current nor its next configuration") + ) + continue + out.live += 1 + out.keys_fps.add(b.keys_fp) + out.scim = out.scim or bool(fields.get("scim")) + return out + + +def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> None: + if stopped_unset == "wait": + out.aging.append(b.instance_id) + return + out.refusals.append( + Refusal( + f"process {b.instance_id}", + f"{what} at {b.last_beat}, and may still be writing (its drain and retirement " + "after its stop beat)", + ) + ) + + +def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: + out = [] + if not verdict.live and not verdict.booting: + out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) + if len(verdict.keys_fps) > 1: + out.append( + Refusal( + "processes", + "the live processes report different root-secret fingerprints (keys_fp): " + "finish the root-secret rollout first", + ) + ) + if verdict.scim and not scim_ok: + out.append( + Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") + ) + return out + + +async def settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + *, + extend_s: float = 0.0, +) -> Verdict: + """Read and judge until nothing is waited on: Refused at the first refusal, and once + `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended + once, to `extend_s` past the first sight of the last process first seen booting.""" + first_seen: dict[str, float] = {} + extended = False + while True: + verdict = judge_beats(await read()) + if verdict.refusals: + raise Refused(verdict.refusals) + if not verdict.booting and not verdict.aging: + if refusals := final_refusals(verdict): + raise Refused(refusals) + return verdict + for i in verdict.booting: + first_seen.setdefault(i, wait.spent) + if await wait.pause(): + continue + if extend_s and not extended and verdict.booting: + extended = True + wait.seconds = max(wait.seconds, max(first_seen[i] for i in verdict.booting) + extend_s) + if await wait.pause(): + continue + raise Refused( + [Refusal(f"process {i}", "still booting at the end of --wait") for i in verdict.booting] + + [ + Refusal(f"process {i}", "stopped with LOOM_CELL unset, not aged out within --wait") + for i in verdict.aging + ] + ) + + +# -- the rescan's watch ---------------------------------------------------------------- + +_READ_TIMEOUT_S = 30.0 # not a setting: one heartbeat read's bound +_SKEW_S = 10.0 # not a setting: a margin for clock skew between ClickHouse hosts + + +def watch_period_s() -> float: + """`T'`: how long a row whose window reaches past the rescan's start is kept before the + leader's sweep may drop it, less a margin for clock skew.""" + return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + + +class Watch: + """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's + start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at + the end even once the leader swept its rows.""" + + def __init__( + self, ch: Any, since: str, name: str, clock: Callable[[], float] = time.monotonic + ) -> None: + self.ch, self.since, self.name = ch, since, name + self._clock = clock + self.cell_ids: set[str] = set() + self.unset: dict[str, str] = {} + self.no_cell: dict[str, str] = {} + self.gap = False + self._good = clock() + + def refusals_up_front(self) -> list[Refusal]: + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + async def read_once(self) -> list[Beat]: + try: + async with asyncio.timeout(_READ_TIMEOUT_S): + beats = await read_beats(self.ch, since=self.since) + except Exception: + self._check_gap() + raise + self._check_gap() + self._good = self._clock() + for b in beats: + if b.cell == self.name: + self.cell_ids.add(b.instance_id) + elif b.cell is not None: + self.unset[b.instance_id] = b.last_beat + elif b.stopped: + self.no_cell[b.instance_id] = b.last_beat + return beats + + def _check_gap(self) -> None: + if self._clock() - self._good > watch_period_s(): + self.gap = True + + async def run(self) -> None: + """Read until cancelled; a failed read is retried at the next period.""" + while True: + await asyncio.sleep(watch_period_s() / 2) + try: + await self.read_once() + except Exception: # noqa: BLE001 — a failed read is retried; a gap refuses + continue + + def refusals(self) -> list[Refusal]: + out = [ + Refusal( + f"process {i}", + f"ran with LOOM_CELL unset or another cell during the rescan (last beat {at})", + ) + for i, at in sorted(self.unset.items()) + ] + out += [ + Refusal( + f"process {i}", + f"stopped during the rescan before reporting its LOOM_CELL (last beat {at})", + ) + for i, at in sorted(self.no_cell.items()) + if i not in self.cell_ids + ] + if self.gap: + out.append( + Refusal( + "heartbeats", + "not read for longer than the leader keeps a stopped process's rows: " + "rerun attach", + ) + ) + return out diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py new file mode 100644 index 0000000000..6ee4432907 --- /dev/null +++ b/server/cli/cell_attach_pg.py @@ -0,0 +1,690 @@ +"""The first attach's reads and writes on the cell's Postgres (docs/architecture/cells.md, +"Attaching a cell"): the session checks, the one-run lock, the catalog checks attach and +`loom doctor` share (the roles that can make an org held or lower the mode, the settings +and subscriptions that pass an unchecked writer), the trigger pass, the mode commit, its +durable read and the wait for the transactions that could have read `recording`. + +Every check returns its refusals as `cell_ch.Refusal` (subject, reason), as the org scan +and the ClickHouse writer check do, so attach and doctor print the same names.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, cell_triggers, pg +from server.core.cell import ATTACH_LOCK_KEY + +_sleep = asyncio.sleep # the seam the wait tests replace + +# The pause between polls of a wait. +POLL_S = 1.0 # not a setting: a poll interval +# The migration lock's wait a trigger try (a boot holds it for its migrations). +_MIGRATION_LOCK_TIMEOUT = "10s" +# The mode commit's lock wait a try, and the statement bound above it. +_MODE_LOCK_TIMEOUT = "2s" +_MODE_STATEMENT_TIMEOUT = "5s" +# The server ends the lock session soon after its client is gone, releasing the lock. +_KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} + + +class Wait: + """`--wait`: the seconds attach's waits share, counting only the time spent waiting + (never scan time).""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self.spent = 0.0 + self._clock = clock + + def left(self) -> float: + return max(0.0, self.seconds - self.spent) + + def now(self) -> float: + return self._clock() + + async def pause(self, seconds: float = POLL_S) -> bool: + """Wait `seconds` (at most what is left) and count them; False once none is left.""" + step = min(seconds, self.left()) + if step <= 0: + return False + start = self._clock() + await _sleep(step) + self.spent += max(step, self._clock() - start) + return True + + +def _split(key: int) -> str: + """The `pg_locks` columns that hold a bigint advisory key, in this database.""" + return ( + f"l.locktype = 'advisory' AND l.classid = ({key} >> 32)::oid " + f"AND l.objid = ({key} & 4294967295)::oid AND l.objsubid = 1 " + "AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())" + ) + + +_LOCK_HOLDER = f""" +SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted +""" + +_IDENTITY = ( + "SELECT pg_backend_pid() AS pid, backend_start FROM pg_stat_activity " + "WHERE pid = pg_backend_pid()" +) + + +async def direct_refusals(conn: Any, what: str) -> list[Refusal]: + """Whether `conn` is one direct, marked session: the same backend across two + transactions, and the tenancy marker kept through `RESET ALL` (a pooler that strips + startup parameters drops it). Behind a transaction pooler a session-level lock would + outlive the CLI on the pooler's server connection, and attach's tenancy rows would be + refused as an unchecked writer's.""" + async with conn.transaction(): + first = await conn.fetchrow(_IDENTITY) + async with conn.transaction(): + second = await conn.fetchrow(_IDENTITY) + out = [] + if first is None or second is None or tuple(first) != tuple(second): + out.append( + Refusal( + "postgres session", + f"{what}'s session moved between backends: point LOOM_PG_DSN straight at " + "Postgres, never through a pooler", + ) + ) + await conn.execute("RESET ALL") # a startup value survives it, a SET does not + if await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) != "on": + out.append( + Refusal( + "postgres session", + f"{what}'s session loses {pg.TENANCY_MARKER}: run it with LOOM_CELL set, " + "LOOM_PG_DSN straight at Postgres", + ) + ) + return out + + +_HELD = f""" +SELECT EXISTS (SELECT 1 FROM pg_locks l + WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted AND l.pid = pg_backend_pid()) +""" + + +async def lock_holder(conn: Any) -> str: + """Who holds the attach lock ('' when no one), never a query's text.""" + rows = await conn.fetch(_LOCK_HOLDER) + return "; ".join( + f"pid {r['pid']} ({r['application_name'] or 'no name'}, from {r['client'] or 'local'}, " + f"since {r['backend_start']})" + for r in rows + ) + + +async def take_lock(conn: Any, what: str) -> None: + """Take the attach lock on `conn` for the session's life, confirmed held by this + backend; Refused naming its holder when another run holds it.""" + for name, value in _KEEPALIVE.items(): + await conn.execute(f"SET {name} = {value}") + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", ATTACH_LOCK_KEY): + holder = await lock_holder(conn) or "a session that just ended" + raise Refused( + [ + Refusal( + "attach lock", + f"held by {holder}: an attach, prepare or update is running; {what} " + "runs one at a time", + ) + ] + ) + if not await lock_held(conn): + raise Refused( + [ + Refusal( + "attach lock", + "taken but not held by this session's backend: a pooler between " + "this command and Postgres; point LOOM_PG_DSN straight at it", + ) + ] + ) + + +async def lock_held(conn: Any) -> bool: + """Whether `conn`'s backend holds the attach lock.""" + return bool(await conn.fetchval(_HELD)) + + +async def encoding_refusals(conn: Any) -> list[Refusal]: + """A cell's database is UTF8: its `text` then holds no NUL or invalid UTF-8, so every + tenancy row within the length limit is an org a request can name.""" + encoding = await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()" + ) + if encoding == "UTF8": + return [] + return [ + Refusal( + "postgres database", + f"its encoding is {encoding}, not UTF8: a cell's database is UTF8 " + "(recreate it with ENCODING 'UTF8' and restore into it)", + ) + ] + + +# -- who can make an org held, or lower the mode -------------------------------------- + +_SYSTEM_SCHEMA = ( + "n.nspname NOT IN ('pg_catalog', 'information_schema') " + "AND n.nspname NOT LIKE 'pg\\_toast%' AND n.nspname NOT LIKE 'pg\\_temp\\_%'" +) + +# Every right a login role may hold that lets it make an org held, lower the mode, or run +# its own SQL as the servers on their marked sessions: each row a role and why. +_RIGHTS = f""" +WITH owned AS ( + SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + UNION ALL + SELECT p.proowner, 'owns loom.' || p.proname || '()' FROM pg_proc p + WHERE p.oid IN (to_regprocedure('loom.tenancy_refuse_unheld()'), + to_regprocedure('loom.tenancy_refuse_unmarked()'), + to_regprocedure('loom.cell_first_immutable()')) + UNION ALL + SELECT d.datdba, 'owns the database' FROM pg_database d + WHERE d.datname = current_database() + UNION ALL + SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} +), +login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) +SELECT r.rolname AS role, w.why FROM login r, LATERAL ( + SELECT 'may insert into loom.tenancy' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') + UNION ALL SELECT 'may update loom.tenancy (org)' + WHERE has_column_privilege(r.oid, 'loom.tenancy', 'org', 'UPDATE') + UNION ALL SELECT 'may update loom.tenancy_mode (mode)' + WHERE has_column_privilege(r.oid, 'loom.tenancy_mode', 'mode', 'UPDATE') + UNION ALL SELECT 'may insert into loom.tenancy_mode' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy_mode', 'INSERT') + UNION ALL SELECT 'holds TRIGGER on loom.' || c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND has_table_privilege(r.oid, c.oid, 'TRIGGER') + UNION ALL SELECT 'may create in the database' + WHERE has_database_privilege(r.oid, current_database(), 'CREATE') + UNION ALL SELECT 'may create in schema ' || n.nspname FROM pg_namespace n + WHERE {_SYSTEM_SCHEMA} AND has_schema_privilege(r.oid, n.oid, 'CREATE') + UNION ALL SELECT 'may set session_replication_role' + WHERE has_parameter_privilege(r.oid, 'session_replication_role', 'SET') + OR has_parameter_privilege(r.oid, 'session_replication_role', 'ALTER SYSTEM') + UNION ALL SELECT 'may ALTER SYSTEM SET {pg.TENANCY_MARKER}' + WHERE has_parameter_privilege(r.oid, '{pg.TENANCY_MARKER}', 'ALTER SYSTEM') + UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g + WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') + AND pg_has_role(r.oid, g.oid, 'MEMBER') + UNION ALL SELECT o.why || CASE WHEN o.owner = r.oid THEN '' + ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' + END + FROM owned o WHERE pg_has_role(r.oid, o.owner, 'MEMBER') +) w(why) +ORDER BY 1, 2 +""" + +# The roles attach counts as its own: superusers, and a login role that itself owns +# loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the +# triggers, or lower the mode, whatever else it holds). +_OWN_ROLES = """ +SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( + SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) +)) OR rolname = current_user +ORDER BY 1 +""" + + +@dataclass(frozen=True) +class Holders: + """The login roles holding a right `_RIGHTS` counts: the ones attach counts as its own + (`own`, named in its output) and every other with its reasons.""" + + own: list[str] + others: dict[str, list[str]] + + +async def privilege_holders(conn: Any) -> Holders: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + others: dict[str, list[str]] = {} + for r in await conn.fetch(_RIGHTS): + if r["role"] not in own: + others.setdefault(str(r["role"]), []).append(str(r["why"])) + return Holders(own, others) + + +def privilege_refusals(holders: Holders, allowed: Iterable[str]) -> list[Refusal]: + allow = set(allowed) + return [ + Refusal( + f"postgres role {role}", + f"{'; '.join(why)}: it can make an org held or lower the mode outside attach; " + "revoke it, or name the role with --pg-allow-role", + ) + for role, why in sorted(holders.others.items()) + if role not in allow + ] + + +# -- what passes an unchecked writer through the triggers ----------------------------- + +_ROLE_SETTINGS = f""" +SELECT coalesce(r.rolname, 'every role') AS role, coalesce(d.datname, 'every database') AS db, + cfg +FROM pg_db_role_setting s +LEFT JOIN pg_roles r ON r.oid = s.setrole +LEFT JOIN pg_database d ON d.oid = s.setdatabase, +LATERAL unnest(s.setconfig) cfg +WHERE s.setdatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database())) + AND lower(split_part(cfg, '=', 1)) IN ('{pg.TENANCY_MARKER}', 'session_replication_role') +ORDER BY 1, 2, 3 +""" + +_SUBSCRIPTIONS = """ +SELECT subname, subenabled FROM pg_subscription +WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) +ORDER BY 1 +""" + + +async def unmarked_refusals(conn: Any, session_passes: bool) -> list[Refusal]: + """What lets a session that sends no marker pass the triggers: the session read + (`session_passes`, doctor's unmarked session), an `ALTER ROLE … SET` or + `ALTER DATABASE … SET` of the marker or `session_replication_role` for this database or + every one, and any logical-replication subscription into this database (its apply + worker runs as `replica`, firing no origin trigger).""" + out = [] + if session_passes: + out.append( + Refusal( + "postgres unmarked session", + "a session that sends no marker is marked, or runs as replica (a role, " + "database, ALTER SYSTEM or options setting): every unchecked writer passes", + ) + ) + for r in await conn.fetch(_ROLE_SETTINGS): + out.append( + Refusal( + f"postgres setting for {r['role']} in {r['db']}", + f"{r['cfg']}: an unchecked writer under it passes the triggers; reset it", + ) + ) + for r in await conn.fetch(_SUBSCRIPTIONS): + state = "enabled" if r["subenabled"] else "disabled" + out.append( + Refusal( + f"postgres subscription {r['subname']}", + f"{state}: its apply worker fires no origin trigger; drop it", + ) + ) + return out + + +async def session_passes() -> bool: + """Whether a session that sends no marker passes the triggers all the same: marked, + or in the replica role that fires no origin trigger.""" + async with pg.dedicated(statement_timeout_ms=5000, marked=False) as conn: + value = await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) + role = await conn.fetchval("SELECT current_setting('session_replication_role')") + return bool(value == "on" or role == "replica") + + +# -- attach's own role ------------------------------------------------------------------ + +_OWN_GRANTS = """ +SELECT need, held FROM (VALUES + ('membership in pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER')), + ('membership in pg_signal_backend', pg_has_role(current_user, 'pg_signal_backend', 'MEMBER')), + ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), + ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), + ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), + ('UPDATE on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'UPDATE')), + ('SELECT on loom.cell_self', has_table_privilege('loom.cell_self', 'SELECT')), + ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), + ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), + ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', + has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) +) v(need, held) +WHERE NOT held +""" + + +async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: + """What attach's own role lacks (the guarded tables' owner roles are `missing`'s + `owner` defects), and a server that allows prepared transactions.""" + role = await conn.fetchval("SELECT current_user") + lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] + for table in tables: + if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): + lacks.append(f"SELECT on {table}") + out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] + out += await prepared_refusals(conn) + return out + + +async def prepared_refusals(conn: Any) -> list[Refusal]: + setting = await conn.fetchval("SELECT current_setting('max_prepared_transactions')::int") + if setting: + return [ + Refusal( + "postgres max_prepared_transactions", + f"is {setting}: a prepared transaction outlives every session, so step 4 " + "could not wait for it; set it to 0", + ) + ] + return [] + + +# -- the triggers ----------------------------------------------------------------------- + +_MIGRATION_LOCK = "SELECT pg_advisory_lock($1)" +_MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" + + +async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: + """Give every planned table its triggers (`cell_triggers.ensure_table` with + `recording_first`), one try at a time under the migration lock, retrying a table whose + lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield + by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + deadlock_ms = int( + await conn.fetchval( + "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " + "FROM pg_settings WHERE name = 'deadlock_timeout'" + ) + ) + out: list[Refusal] = [] + for table in await cell_triggers.planned_tables(conn): + yielded = False + while True: + lock_timeout = cell_triggers.LOCK_TIMEOUT + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_timeout, yielded = f"{deadlock_ms + 500}ms", True + done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) + if done: + break + if not await wait.pause(): + out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) + break + log(f"loom.{table}: its lock is busy; retrying") + if out: + return out + return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] + + +async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: + await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") + try: + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) == "55P03": + return False # a boot migrating: the next try + raise + finally: + await conn.execute("RESET lock_timeout") + try: + return await cell_triggers.ensure_table( + conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + ) + finally: + await conn.execute(_MIGRATION_UNLOCK, key) + + +def _busy_refusal(table: str, holders: list[Any]) -> Refusal: + named = "; ".join( + f"pid {h['pid']} ({'holds' if h['granted'] else 'waiting'}, {h['backend_type']}, " + f"{h['application_name'] or 'no name'}, transaction since {h['xact_start']}" + + (", an anti-wraparound vacuum, which never yields" if h["wraparound"] else "") + + ")" + for h in holders + ) + return Refusal( + f"postgres loom.{table}", + f"its triggers' lock was not taken within --wait; held by {named or 'no one now'}", + ) + + +# -- step 3: the mode commit ------------------------------------------------------------ + +# The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on +# loom.tenancy_mode in this database, which only attach takes. +_MODE_HOLDERS = """ +SELECT l.pid, a.application_name, a.xact_start, r.rolsuper, l.granted +FROM pg_locks l +JOIN pg_stat_activity a ON a.pid = l.pid +LEFT JOIN pg_roles r ON r.oid = a.usesysid +WHERE l.locktype = 'relation' AND l.relation = 'loom.tenancy_mode'::regclass + AND l.mode = 'ExclusiveLock' + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND l.pid <> pg_backend_pid() +""" + +_TIMED_OUT = frozenset({"55P03", "57014"}) + + +async def commit_enforced( + conn: Any, + confirm_lock: Callable[[], Any], + ready: Callable[[], Any], + wait: Wait, + log: Callable[[str], None], +) -> None: + """Raise the mode to `enforced` on `conn` (a session of its own, never the lock + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, + within `wait`; `confirm_lock` before each try and each termination (it refuses when + another run holds the attach lock), and `ready` before each try (False skips the try + while a process boots). After a failed try the in-flight mode transaction, found by its + lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the + durable read's to say.""" + while True: + await confirm_lock() + if await ready(): + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", f"the mode update read {status}" + ) + ] + ) + return + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + log("the mode's lock is busy; retrying") + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads it " + "FOR SHARE; rerun attach", + ) + ] + ) + + +async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: + """End an earlier run's mode transaction still in flight and wait until its lock + clears; Refused when its role is a superuser and attach's is not.""" + for _ in range(30): + rows = await conn.fetch(_MODE_HOLDERS) + if not rows: + return + superuser = bool( + await conn.fetchval("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") + ) + for r in rows: + if r["rolsuper"] and not superuser: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"an earlier mode transaction ({r['application_name'] or 'no name'}, " + f"since {r['xact_start']}) runs as a superuser: a superuser must " + "end it (pg_terminate_backend)", + ) + ] + ) + await confirm_lock() + await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) + await _sleep(0.1) + + +# -- the durable read ------------------------------------------------------------------- + +_DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried + + +@dataclass(frozen=True) +class Durable: + """What the durable read saw: the mode, the `first` row, and the xid its logical + message took.""" + + mode: str + first: tuple[bool, str] | None + xid: int + + +async def durable_read() -> Durable: + """The mode and the `first` row read in a transaction that emits a transactional + logical message, whose commit waits for the synchronous standbys: once it returns, the + mode read is on them. Refused when the message took no xid.""" + async with asyncio.timeout(_DURABLE_TRY_S), pg.durable_transaction() as conn: + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") + xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + +async def snapshot_xmax(conn: Any) -> int: + return int(await conn.fetchval("SELECT pg_snapshot_xmax(pg_current_snapshot())::text")) + + +# -- step 4: the transactions that could have read `recording` -------------------------- + +# Every backend but this statement's own: of the cell's database, one whose xid or (but +# an autovacuum's) snapshot is older than X; of another database, one whose xid is older +# (it holds every snapshot's horizon). Both compared through age() against one reference, +# which is exactly `< X`, wraparound-safe, and takes no xid. +_OPEN = """ +WITH x AS (SELECT age($1::text::xid8::xid) AS a) +SELECT s.pid, s.backend_type, s.application_name, s.xact_start::text AS xact_start, + s.datid = (SELECT oid FROM pg_database WHERE datname = current_database()) AS ours, + coalesce(s.backend_xid IS NOT NULL AND age(s.backend_xid) > x.a, false) AS xid_old, + coalesce(s.backend_xmin IS NOT NULL AND age(s.backend_xmin) > x.a + AND s.backend_type <> 'autovacuum worker', false) AS xmin_old, + coalesce(s.query = '', false) AS hidden +FROM pg_stat_activity s, x +WHERE s.pid <> pg_backend_pid() +""" + + +def _named(r: Any) -> str: + return ( + f"pid {r['pid']} ({r['backend_type']}, {r['application_name'] or 'no name'}, " + f"transaction since {r['xact_start']})" + ) + + +async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: + """Wait until no backend of the cell's database holds an xid or a snapshot older than + `xmax`; Refused at once on a prepared transaction or an unreadable row, and past + `wait` naming each backend waited on and each other database's horizon holder.""" + while True: + if refusals := await prepared_refusals(conn): + raise Refused(refusals) + if await conn.fetchval("SELECT count(*) FROM pg_prepared_xacts"): + raise Refused( + [Refusal("postgres prepared transaction", "one is open: commit or roll it back")] + ) + rows = await conn.fetch(_OPEN, str(xmax)) + hidden = [r for r in rows if r["ours"] and r["hidden"]] + if hidden: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + "its transaction state is hidden from attach's role: grant it " + "pg_read_all_stats", + ) + for r in hidden + ] + ) + waited = [r for r in rows if r["ours"] and (r["xid_old"] or r["xmin_old"])] + if not waited: + return + if not await wait.pause(): + horizon = [r for r in rows if not r["ours"] and r["xid_old"]] + raise Refused( + [ + Refusal(f"postgres {_named(r)}", "may have read `recording`: still open") + for r in waited + ] + + [ + Refusal( + f"postgres {_named(r)}", + "another database's transaction, holding the snapshot horizon", + ) + for r in horizon + ] + ) + + +# -- the tenancy rows ------------------------------------------------------------------- + + +async def write_tenancy(orgs: Iterable[str]) -> int: + """The tenancy rows of `orgs`, on a marked session, as one statement per thousand; the + rows written.""" + batch = sorted(orgs) + written = 0 + for i in range(0, len(batch), 1000): + status = await pg.execute( + "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", + batch[i : i + 1000], + ) + written += int(status.split()[-1]) + return written + + +async def read_mode(conn: Any) -> tuple[str, tuple[bool, str] | None]: + """The cell's state (`cell.read_state`) and its `first` row.""" + state = await cell.read_state(conn) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + return state, None if first is None else (bool(first["is_first"]), str(first["cell"])) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 8ad83098a9..cf83fa8f27 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -1,7 +1,10 @@ """`loom cell prepare`: register this cell in the router's directory and write the rows its processes read (docs/architecture/cells.md, "Preparing a cell" and "The tenancy check"). -It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names): +It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names), +holding the attach lock (`cell.ATTACH_LOCK_KEY`) on a direct session of the cell's Postgres, +after that session's checks (one backend, the tenancy marker kept, a UTF8 database), so it +never runs beside an attach or another prepare: 1. It reads the cell's Postgres (deployment id, `first`, mode and own rows, tenancy rows) and the live processes' heartbeats, and refuses a cell whose rows name another cell @@ -23,11 +26,13 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets import sys import time +from collections.abc import AsyncIterator from dataclasses import dataclass from typing import Any @@ -46,6 +51,9 @@ _WAIT_MAX_S = 86400 # not a setting: a flag's bound +# A bound on the lock session's statements: it reads the catalog and holds the lock. +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on one CLI session's reads + _sleep = asyncio.sleep # the seam the retry tests replace @@ -442,6 +450,38 @@ async def prepare( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, +) -> int: + """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" + async with attach_lock(): + return await _prepare_locked(ch, args, router, transport) + + +@contextlib.asynccontextmanager +async def attach_lock() -> AsyncIterator[None]: + """The attach lock, held on a direct session of the cell's Postgres while the block + runs, after that session's checks: prepare and attach write the same probe nonce, and + one of them runs at a time.""" + from server.cli import cell_attach_pg + from server.cli.cell_ch import Refused as Stopped + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + try: + refusals = await cell_attach_pg.direct_refusals(conn, "prepare") + refusals += await cell_attach_pg.encoding_refusals(conn) + if refusals: + raise Stopped(refusals) + await cell_attach_pg.take_lock(conn, "prepare") + except Stopped as e: + raise Refused(str(e)) from None + yield + + +async def _prepare_locked( + ch: Any, + args: argparse.Namespace, + router: httpx.URL, + transport: httpx.AsyncBaseTransport | None, ) -> int: rows = await _read_rows() if not rows.deployment_id: diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index 6968dd49b0..e7d9077a02 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -67,6 +67,19 @@ } ) +# Tables a worker drains into another table, deleting what it moved: each is scanned +# before every table that is not one, so a row moved mid-scan is read where it was or where +# it went. A table a materialized view reads is ordered the same way, from +# `system.tables` dependencies (`order_sources`). +MOVE_SOURCES = frozenset( + { + "loom.learnings_queue", + "loom.memory_code_link_queue", + "loom.memory_extraction_queue", + "loom.memory_webhook_outbox", + } +) + # A refreshable view's temporary table while a refresh exchanges its target. _EXCHANGE = re.compile(r"^\.tmp\.inner(_id)?\.") # A provider transition's copies (server/erasure/store/provider_copies.py). @@ -338,6 +351,32 @@ async def list_postgres(conn: Any) -> tuple[list[str], list[Refusal]]: return names, refusals +def order_sources(tables: list[Table], view_sources: Iterable[str]) -> list[Table]: + """`tables` with the move sources (`MOVE_SOURCES`, and every table a view reads, + `view_sources`, canonical names) first, each group by name.""" + first = MOVE_SOURCES | set(view_sources) + return sorted(tables, key=lambda t: (t.canonical not in first, t.canonical)) + + +async def view_sources( + ch: Any, topology: Topology, names: Mapping[str, str], dbs: list[str] +) -> set[str]: + """The tables of `dbs` a view depends on, as canonical names.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, database, name FROM {source:tables} " + "WHERE database IN {dbs:Array(String)} AND notEmpty(dependencies_table)", + {"dbs": dbs}, + ) + return { + f"{ch_names.logical(str(r['database']))}.{r['name']}" + for found in rows.values() + for r in found + } + + async def list_tables( ch: Any, pg_conn: Any, topology: Topology, names: Mapping[str, str] ) -> TableList: @@ -370,7 +409,8 @@ async def _list_tables( if copies := provider_copies(tables): out.refusals += copies return out - out.clickhouse, problems = judge_clickhouse(tables, classified, topology) + scanned, problems = judge_clickhouse(tables, classified, topology) + out.clickhouse = order_sources(scanned, await view_sources(ch, topology, names, databases())) out.refusals += problems out.refusals += await reverse_clickhouse(ch, tables, classified) out.postgres, problems = await list_postgres(pg_conn) @@ -460,7 +500,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def _pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -615,7 +655,7 @@ async def _read_scan( return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def _ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) @@ -697,10 +737,10 @@ async def scan( result.add("configured", org) with query_tag(TAG): for name in tables.postgres: - for org in await _pg_orgs(name, budget): + for org in await pg_orgs(name, budget): result.add(f"postgres {name}", org) for table in tables.clickhouse: - for raw in await _ch_orgs(ch, table, topology, budget): + for raw in await ch_orgs(ch, table, topology, budget): result.add(f"clickhouse {table.canonical}", raw) result.elapsed_s = budget.elapsed() return result diff --git a/server/cli/doctor.py b/server/cli/doctor.py index d1295ffb32..29a9ddabf6 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -566,10 +566,15 @@ def grade(p: Any) -> str: return [(f"problem:{p.code}", grade(p), f"{p.message} Fix: {p.fix}") for p in problems] -def _run_doctor_checks() -> list[tuple[str, str, str]]: +def _run_doctor_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The live checks (Python, settings, JWT, ClickHouse, LLM) followed - by the shared problem list, as (name, grade, detail).""" - return _run_live_checks() + _code_mirror_checks() + _problem_checks(_doctor_problems()) + by the shared problem list, as (name, grade, detail). `allowed_roles` is + `--pg-allow-role`, for a cell's checks.""" + return ( + _run_live_checks(allowed_roles) + + _code_mirror_checks() + + _problem_checks(_doctor_problems()) + ) def _code_mirror_checks() -> list[tuple[str, str, str]]: @@ -582,7 +587,7 @@ def _code_mirror_checks() -> list[tuple[str, str, str]]: return [("code_mirrors", WARN, f"not checked: {type(e).__name__}: {e}")] -def _run_live_checks() -> list[tuple[str, str, str]]: +def _run_live_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """Run the doctor's live checks and return them as (name, grade, detail) with grade one of PASS / WARN / FAIL. @@ -664,7 +669,7 @@ def _run_live_checks() -> list[tuple[str, str, str]]: if settings_ok and ch_ok and settings.cell: from server.cli import doctor_cell - checks += doctor_cell.cell_checks() + checks += doctor_cell.cell_checks(allowed_roles) # 5. LLM provider resolution: what extraction runs on and # what decided it; WARN when the provider was inferred (or named) but its @@ -677,6 +682,17 @@ def _run_live_checks() -> list[tuple[str, str, str]]: return checks +def _stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`--stray-orgs`, on a cell only (LOOM_CELL set).""" + from server.settings import settings + + if not settings.cell: + return [("cell_stray_orgs", WARN, "not read: LOOM_CELL is unset (no cell)")] + from server.cli import doctor_cell + + return doctor_cell.stray_org_checks(budget_s) + + def _client_checks(args: argparse.Namespace) -> list[tuple[str, str, str]]: """`loom doctor --client`: the checks from the machine a coding agent runs on, with the CLI's own HTTP helpers. A crash in one of @@ -722,7 +738,9 @@ def _cmd_doctor(_args: argparse.Namespace) -> int: operator sees the full picture. Exit 0 = nothing failed (warnings are listed but do not fail); 1 = at least one FAIL. """ - checks = _run_doctor_checks() + checks = _run_doctor_checks(tuple(getattr(_args, "pg_allow_role", None) or ())) + if getattr(_args, "stray_orgs", False): + checks += _stray_org_checks(_args.stray_orgs_budget) # The client's side of the same picture, when asked. client = _client_checks(_args) if getattr(_args, "client", False) else [] checks = _client_machine_grades(checks, client) + client @@ -841,4 +859,21 @@ def _add_doctor_flags(p_doc: argparse.ArgumentParser) -> None: ), ) p_doc.add_argument("--server-url", default=None, help="Override the client's server URL.") + p_doc.add_argument( + "--pg-allow-role", + action="append", + default=[], + help="On a cell: a Postgres role that may make an org held (the servers' role).", + ) + p_doc.add_argument( + "--stray-orgs", + action="store_true", + help="On a cell: also read the orgs its stores hold without a tenancy row.", + ) + p_doc.add_argument( + "--stray-orgs-budget", + type=float, + default=60.0, + help="Seconds --stray-orgs may read for, smallest tables first (default 60).", + ) p_doc.add_argument("--token", default=None, help="Override the client's token.") diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index f2d6f80367..c79723bbbc 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -1,7 +1,11 @@ """`loom doctor`'s cell checks, run only with LOOM_CELL set (docs/architecture/cells.md): the deployment check, a partial cell state, the tenancy marker on this process's -sessions and on every session, and the defects of the org triggers and the `loom.tenancy` -guard (`cell_triggers.missing`). +sessions and on every session, the defects of the org triggers and the `loom.tenancy` +guard (`cell_triggers.missing`), and the point-in-time reads attach makes, run again on +every doctor run: the login roles that can make an org held or lower the mode, and the +role and database settings and subscriptions that pass an unchecked writer +(`cell_attach_pg`). `--stray-orgs` also reads the orgs the cell's stores hold without a +tenancy row (`stray_org_checks`). A refusal (ClickHouse access denied, no LOOM_PG_DSN, another recorded authority or a Postgres of another deployment) fails the doctor's exit and reads no cell state; @@ -78,14 +82,34 @@ async def _unmarked_session() -> bool: """Whether a session that sends no marker passes the triggers all the same (a role, database, ALTER SYSTEM or options setting): marked, or in the replica role that fires no origin trigger.""" + from server.cli import cell_attach_pg + + return await cell_attach_pg.session_passes() + + +async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, str]]: + """Attach's point-in-time Postgres reads, again: the login roles outside the owner + roles, the superusers and `--pg-allow-role` that can make an org held or lower the + mode, and the settings and subscriptions that pass an unchecked writer.""" + from server.cli import cell_attach_pg from server.core import pg - async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS, marked=False) as conn: - value: str | None = await conn.fetchval( - "SELECT current_setting($1, true)", pg.TENANCY_MARKER - ) - role: str = await conn.fetchval("SELECT current_setting('session_replication_role')") - return value == "on" or role == "replica" + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + holders = await cell_attach_pg.privilege_holders(conn) + unchecked = await cell_attach_pg.unmarked_refusals(conn, False) + roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) + return [ + ( + "cell_tenancy_privilege", + WARN if roles else PASS, + "; ".join(str(r) for r in roles) or "no role outside the owner roles can", + ), + ( + "cell_unchecked_writers", + WARN if unchecked else PASS, + "; ".join(str(r) for r in unchecked) or "no setting or subscription passes one", + ), + ] def _grades( @@ -149,7 +173,7 @@ def _grades( return out -async def _checks(ch: Any) -> list[tuple[str, str, str]]: +async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: try: if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): return [("cell_state", PASS, f"cell checks not run: {before}")] @@ -159,7 +183,11 @@ async def _checks(ch: Any) -> list[tuple[str, str, str]]: return [("cell_deployment", FAIL, f"deployment check refused: {e}")] except Exception as e: # noqa: BLE001 — an unreadable cell is named, not a failed run return [("cell_state", WARN, f"not read: {_reason(e)}")] - return _grades(state, marked, marks_all, defects) + try: + reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) + except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run + reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + return _grades(state, marked, marks_all, defects) + reads def _reason(e: BaseException) -> str: @@ -167,9 +195,10 @@ def _reason(e: BaseException) -> str: return f"{type(e).__name__}: {str(e) or 'no answer within the bound'}" -def cell_checks() -> list[tuple[str, str, str]]: +def cell_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The cell checks, on the CLI's ClickHouse client connected for them alone (no CLI - deployment check: a deployment not booted yet is named, not refused).""" + deployment check: a deployment not booted yet is named, not refused). `allowed_roles` + is doctor's `--pg-allow-role`.""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client @@ -180,10 +209,121 @@ async def _go() -> list[tuple[str, str, str]]: connect_declared(ch, org_free=reads("loom doctor reads a cell's state")), timeout=_SESSION_S, ) - return await _checks(ch) + return await _checks(ch, allowed_roles) except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: await disconnect(ch) return asyncio.run(_go()) + + +# -- --stray-orgs -------------------------------------------------------------------------- + + +async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: + """Each ClickHouse table's active row count, from `system.parts` (KeeperMap: 0).""" + from server.core import ch_names + + rows = await ch.query_all( + "SELECT database, table, sum(rows) AS n FROM system.parts WHERE active " + "AND database IN {dbs:Array(String)} GROUP BY database, table", + {"dbs": sorted({t.database for t in tables})}, + ) + return {f"{ch_names.logical(str(r['database']))}.{r['table']}": int(r["n"]) for r in rows} + + +async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: + from server.cli import cell_scan + from server.cli.cell_ch import Refused, databases, read_topology + from server.core import pg + from server.core.cell_triggers import ORG_TABLES + + topology = await read_topology(ch) + if topology.multi: + from server.cli.cell_writers import host_names + + names = await host_names(ch, topology) + else: + names = {} + listed, problems = await cell_scan.list_clickhouse(ch, topology, names, databases()) + ch_tables = [t for t in listed if t.engine.endswith("MergeTree") or t.engine == "KeeperMap"] + sizes = await _table_sizes(ch, ch_tables) + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + held = {str(r["org"]) for r in await conn.fetch("SELECT org FROM loom.tenancy")} + pg_rows = await conn.fetch( + "SELECT 'loom.' || c.relname AS name, greatest(c.reltuples, 0)::bigint AS n " + "FROM pg_class c WHERE c.oid IN (SELECT to_regclass('loom.' || t.relname) FROM (" + + ORG_TABLES + + ") t)" + ) + # Smallest first: a budget that runs out leaves the largest unchecked. + work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] + work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] + work.sort(key=lambda w: (w[0], str(w[2]))) + budget = cell_scan.Budget(budget_s) + stray: dict[str, list[str]] = {} + unchecked: list[str] = [] + for i, (_n, store, table) in enumerate(work): + found = cell_scan.ScanResult() + name = table if store == "postgres" else table.canonical + try: + if store == "postgres": + for org in await cell_scan.pg_orgs(table, budget): + found.add(name, org) + else: + for raw in await cell_scan.ch_orgs(ch, table, topology, budget): + found.add(name, raw) + except Refused: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) + bad = [m for ms in found.malformed.values() for m in ms] + if orgs or bad: + log = " (a log table: never placed)" if name in cell_scan.LOG_TABLES else "" + stray[f"{store} {name}{log}"] = orgs[:5] + bad[:5] + out = [ + ( + "cell_stray_orgs", + WARN if stray else PASS, + "; ".join(f"{t}: {', '.join(o)}" for t, o in sorted(stray.items())) + or "every org found has a tenancy row", + ) + ] + if unchecked or problems: + out.append( + ( + "cell_stray_orgs_unchecked", + WARN, + f"not read within --stray-orgs-budget ({budget_s:g} s): " + + ", ".join(unchecked + [str(p) for p in problems]), + ) + ) + return out + + +def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row + (log-only orgs, names no request can carry, rows an unchecked writer left), smallest + tables first within one budget, naming every table left unchecked. Never a FAIL: the + stray-org erase command removes what it names.""" + from server.cli._ch import connect_declared, disconnect + from server.core.ch import get_client, statement_timeout + + async def _go() -> list[tuple[str, str, str]]: + ch = get_client() + try: + await asyncio.wait_for( + connect_declared(ch, org_free=reads("loom doctor reads a cell's orgs")), + timeout=_SESSION_S, + ) + if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): + return [("cell_stray_orgs", PASS, f"not read: {before}")] + return await _stray_reads(ch, budget_s) + except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run + return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] + finally: + await disconnect(ch) + + with statement_timeout(budget_s): + return asyncio.run(_go()) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 19cf492b0e..9c2c9755ae 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -36,7 +36,9 @@ CELL_FIRST_TRIGGERS = ("cell_first_immutable", "cell_first_no_truncate") # Org-keyed tables the org triggers skip: the cell's own tenancy rows (`tenancy_record` # guards them instead), and the idempotency rows keyed by the caller's token org (a platform -# admin's replay row for its home org is neither refused nor placed). +# admin's replay row for its home org is not refused). Outside the guard is not outside the +# first attach: its org scan (`server/cli/cell_scan.py`) reads every table with an `org` +# column, these included, and places the orgs it finds. EXCLUDED = frozenset({"tenancy", "idempotency", "idempotency_fences"}) # A table's DDL waits this long for its lock a try, LOCK_ATTEMPTS tries LOCK_RETRY_S # apart: a boot never queues behind a long transaction (and every write of the table @@ -77,10 +79,13 @@ # What a trigger's DDL (SHARE ROW EXCLUSIVE) waited behind on a table, read after the # wait failed (pg_blocking_pids is empty by then): the locks held in a mode it conflicts # with (every mode but the two readers take), and the requests queued ahead of it in -# one, whose own blocker heads the queue. +# one, whose own blocker heads the queue. An anti-wraparound autovacuum, which never yields +# its lock, says so in its activity line (compared here, never returned). _HOLDERS = """ SELECT DISTINCT l.pid, l.granted, a.backend_type, a.state, a.application_name, - a.xact_start, a.wait_event + a.xact_start, a.wait_event, + coalesce(a.backend_type = 'autovacuum worker' + AND a.query LIKE '%(to prevent wraparound)', false) AS wraparound FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE l.relation = $1::regclass AND l.pid <> pg_backend_pid() AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) @@ -208,63 +213,107 @@ async def ensure(conn: Any) -> None: tries; a table whose lock it cannot take is logged with the holders of its locks, one the role cannot alter is logged as such, and either is skipped (the boot serves). A failed read of the state or the catalog raises. Runs on the migration session, which - holds the migration lock.""" + holds the migration lock. `ensure_table` runs one table's part alone.""" state = await cell.read_state(conn) if state == "open": return - # A table's statements, in one transaction: one lock wait, and every statement or none. - statements: dict[str, list[str]] = {} first = await _triggers(conn, CELL_FIRST_TRIGGERS) - for name in CELL_FIRST_TRIGGERS: - row = first.get(("cell_first", name)) - if row is not None and row["enabled"] != "A": - statements.setdefault("cell_first", []).append( - f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + always = [ + f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + for name in CELL_FIRST_TRIGGERS + if (row := first.get(("cell_first", name))) is not None and row["enabled"] != "A" + ] + if always: + await _run_ddl(conn, "cell_first", always) + if state != "enforced": + return + tables = await conn.fetch(ORG_TABLES) + for r in tables: + if r["relkind"] == "p": + # No trigger guards it: named here as missing() names it. + log.warning( + "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] ) - if state == "enforced": - present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) - tables = await conn.fetch(ORG_TABLES) - for r in tables: - if r["relkind"] == "p": - # No trigger guards it: named here as missing() names it. - log.warning( - "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] - ) - for r, names in _planned(tables): - table = r["relname"] - for name in names: - row = present.get((table, name)) - if row is None: - statements.setdefault(table, []).append(_create(f"loom.{_ident(table)}", name)) - elif row["enabled"] != "O": - statements.setdefault(table, []).append( - f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}" - ) - for table, sqls in statements.items(): - await _run_ddl(conn, table, sqls) + present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) + for r, names in _planned(tables): + if sqls := _table_statements(r["relname"], names, present): + await _run_ddl(conn, r["relname"], sqls) + + +async def planned_tables(conn: Any) -> list[str]: + """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" + return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + + +async def ensure_table( + conn: Any, + table: str, + *, + recording_first: bool = False, + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its + org triggers), so one table's DDL can run alone: in `enforced`, or with + `recording_first` also in a first cell's `recording`, where the org triggers' function + passes every row and a boot never creates them. False when the DDL was not applied (its + lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); + True when it was, or nothing was needed. Raises for a table outside the planned set, or + in another state.""" + state = await cell.read_state(conn) + if state != "enforced" and not (recording_first and state == "recording"): + raise RuntimeError(f"cell triggers: not created in state {state!r}") + planned = [ + (r, names) for r, names in _planned(await conn.fetch(ORG_TABLES)) if r["relname"] == table + ] + if not planned: + raise ValueError(f"loom.{table} carries no cell triggers") + ((_, names),) = planned + sqls = _table_statements(table, names, await _triggers(conn, names)) + return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + + +def _table_statements( + table: str, names: tuple[str, ...], present: dict[tuple[str, str], Any] +) -> list[str]: + """The DDL that gives `table` its named triggers, each present and enabled.""" + out = [] + for name in names: + row = present.get((table, name)) + if row is None: + out.append(_create(f"loom.{_ident(table)}", name)) + elif row["enabled"] != "O": + out.append(f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}") + return out -async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries. The boot serves past - the two causes cells.md names (its lock not taken, a table this role cannot alter); - any other error fails it, as a failed read of the state does.""" +async def _run_ddl( + conn: Any, + table: str, + sqls: list[str], + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The + boot serves past the two causes cells.md names (its lock not taken, a table this role + cannot alter); any other error fails it, as a failed read of the state does.""" import asyncpg - for attempt in range(LOCK_ATTEMPTS): + for attempt in range(attempts): try: async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return + return True except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return + return False except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. - if attempt < LOCK_ATTEMPTS - 1: + if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue log.warning( @@ -274,13 +323,19 @@ async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: "; ".join(sqls), await _holders(conn, table), ) + return False + + +async def holders(conn: Any, table: str) -> list[Any]: + """The sessions a blocked DDL on `table` waits behind (`_HOLDERS`).""" + return list(await conn.fetch(_HOLDERS, f"loom.{_ident(table)}")) async def _holders(conn: Any, table: str) -> str: """The sessions a blocked DDL waited behind, never their query text (a literal in it may be memory content).""" try: - rows = await conn.fetch(_HOLDERS, f"loom.{_ident(table)}") + rows = await holders(conn, table) except Exception as e: # noqa: BLE001 — naming the holders is best effort log.warning("cell triggers: the lock holders of loom.%s not read: %s", table, e) return "lock holders not read" diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..bb47bfd945 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -53,6 +53,10 @@ _LIVE_INTERVALS = 3 # How often the leader drops the KeeperMap keys of processes gone an hour. _PRUNE_EVERY_S = 600.0 +# How old a row's last beat is before the leader drops its key. A contract across builds, +# not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader +# that sweeps them may run another build, so a change is a design change of its own. +PRUNE_AFTER_S = 3600 # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -130,6 +134,22 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + +def boot_allowance_s() -> float: + """How long a booting process takes to report its first cell row and its first router + answer: two beats and one router read.""" + return 2 * float(settings.instance_heartbeat_seconds) + float(settings.cell_router_timeout_s) + + async def beat(ch: Any, *, stopped: bool = False) -> None: """Write this process's row. A KeeperMap insert replaces the key.""" await ch.insert(TABLE, [row(stopped=stopped)]) @@ -197,6 +217,34 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out +async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose widened window + (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or + past now without one: stopped rows included, which `read_live` drops. `stopped` is its + newest row's flag and `live` whether that row is inside the live window and not + stopped, both judged in ClickHouse.""" + rows = await ch.query_all( + f""" + SELECT instance_id, + argMax(keys_fp, heartbeat_at) AS keys_fp, + argMax(stopped, heartbeat_at) AS stopped, + toString(max(heartbeat_at)) AS last_beat, + max(heartbeat_at) > now64(3) - toIntervalMillisecond({{live:UInt32}}) + AND stopped = 0 AS live + FROM {TABLE} + GROUP BY instance_id + HAVING max(heartbeat_at) + toIntervalMillisecond({{wide:UInt32}}) + > if({{since:String}} = '', now64(3), toDateTime64({{since:String}}, 3)) + """, + params={ + "live": int(live_window_s() * 1000), + "wide": int(widened_window_s() * 1000), + "since": since or "", + }, + ) + return [dict(r) for r in rows] + + def live_rows() -> list[dict[str, Any]]: """The live rows as of the last read; none before the first read or once that read is older than the live window (ClickHouse unreachable).""" @@ -224,14 +272,15 @@ async def _prune(ch: Any) -> None: # Each table's sweep stands alone: one failing leaves the other to run. try: await ch.execute( - "ALTER TABLE loom.instances DELETE WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + "ALTER TABLE loom.instances DELETE " + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("instance row sweep failed: %s", e) try: await ch.execute( "ALTER TABLE loom.cell_heartbeats DELETE " - "WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("cell heartbeat sweep failed: %s", e) @@ -331,8 +380,10 @@ def _reset_for_tests() -> None: __all__ = [ "CELL_TABLE", + "PRUNE_AFTER_S", "TABLE", "beat", + "boot_allowance_s", "cell_beat", "cell_row", "instance_id", @@ -341,8 +392,10 @@ def _reset_for_tests() -> None: "live_window_s", "publish_config", "read_live", + "read_recent", "schema_version", "start", "stop", "tick", + "widened_window_s", ] diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py new file mode 100644 index 0000000000..406d277646 --- /dev/null +++ b/tests/integration/test_cell_attach.py @@ -0,0 +1,635 @@ +"""`loom cell attach` (server/cli/cell_attach.py) against a real Postgres with a synchronous +standby, a real ClickHouse and the router served in process: a prepared first cell in +`recording` with orgs in both stores attaches to `serving` with every org placed and the +triggers created; a rerun on `serving` exits 0; a refusal after step 3 leaves `enforced` and +a rerun resumes from there; a second run is refused at the attach lock; the Postgres reads +attach makes refuse what they name (an unchecked writer's setting, a subscription, a role +that can make an org held, a transaction that may have read `recording`). + +The cell's Postgres and the directory are databases of the test's own; the configured +ClickHouse databases are scratch ones; the cell's processes are heartbeat rows the test +writes. Needs LOOM_PG_DSN (a server with a synchronous standby), LOOM_ROUTER_PG_DSN and +LOOM_CH_HOST.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell as cell_cli +from server.cli import cell_attach, cell_attach_pg, cell_prepare +from server.cli.cell_ch import Refused +from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core.ch import get_client +from server.core.schema import pg_migrate +from server.router import app as router_app +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +OPERATOR = "lco_" + "2" * 128 +CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() +ROUTER = "http://127.0.0.1:7901" +HOST = "loom.example" + + +class _Network(httpx.AsyncBaseTransport): + """What attach and the router reach: the router at ROUTER, the shared host's health + route, the cell's upstream main port (:7777) and its probe listener (any other).""" + + def __init__(self) -> None: + self.listener = httpx.ASGITransport(app=cell_probe.listener_app()) + self.router = httpx.ASGITransport( + app=router_app.build_app(router_app.Router(probe_transport=self)) + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "127.0.0.1" and request.url.port == 7901: + return await self.router.handle_async_request(request) + if request.url.host == HOST or request.url.port == 7777: + return httpx.Response(200, json=cell_probe.answer()) + return await self.listener.handle_async_request(request) + + +async def _admin(sql: str, *args: Any) -> Any: + import asyncpg + + conn = await asyncpg.connect(os.environ["LOOM_PG_DSN"]) + try: + return await conn.execute(sql, *args) + finally: + await conn.close() + + +@pytest.fixture +async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: + cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") + cell_db, own_cell = await create_database(cell_dsn) + router_db, own_router = await create_database(router_dsn) + deployment = f"dep-{uuid.uuid4().hex[:8]}" + tag = f"it-{uuid.uuid4().hex[:8]}-" + for name, value in { + "cell": "first", + "storage_authority": "postgres", + "pg_dsn": own_cell, + "document_s3_bucket": "", + "router_pg_dsn": own_router, + "router_shared_host": HOST, + "router_upstream_probe_url": "http://listener.internal:7790", + "router_upstream_url": "http://upstream.internal:7777", + "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), + "router_operator_token": OPERATOR, + "ch_cluster": "", + "public_base_url": f"https://{HOST}", + "oauth_issuer": "", + }.items(): + monkeypatch.setattr(settings, name, value) + await pg.close_pool() + monkeypatch.setattr(directory, "_pool", None) + # A step's poll interval and the waits stay short. + monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) + ch = get_client() + await ch.connect() + ids: list[str] = [] + try: + await pg_migrate.migrate() + await pg.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) " + "VALUES ('postgres', 1, $1)", + deployment, + ) + await directory.migrate() + await ch.insert( + "loom.storage_authority", + [{"authority": "postgres", "epoch": 10**12, "deployment_id": deployment}], + ) + cell.set_deployment_id(deployment) + yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} + finally: + cell.set_deployment_id("") + await ch.execute( + "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " + "SETTINGS mutations_sync = 2", + {"d": deployment}, + ) + for table in (instances.TABLE, instances.CELL_TABLE): + await ch.execute( + f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", + {"ids": ids}, + ) + await ch.execute( + "DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag} + ) + await ch.disconnect() + await pg.close_pool() + await directory.close() + await drop_database(cell_dsn, cell_db) + await drop_database(router_dsn, router_db) + + +def _value(**keys: Any) -> str: + reported = { + **cell.as_value(cell.mine()), + "serves": True, + "marked": True, + "router": True, + "router_why": "", + **keys, + } + return json.dumps(reported, sort_keys=True) + + +async def _beat( + stores: dict[str, Any], + instance_id: str, + *, + cell_name: str | None = "first", + mode: str = "recording", + stopped: bool = False, + **keys: Any, +) -> None: + """One process's heartbeat rows, as `instances.tick` writes them.""" + ch = stores["ch"] + if instance_id not in stores["ids"]: + stores["ids"].append(instance_id) + await ch.insert( + instances.TABLE, [{**instances.row(stopped=stopped), "instance_id": instance_id}] + ) + if cell_name is not None: + await ch.insert( + instances.CELL_TABLE, + [ + { + "instance_id": instance_id, + "cell": cell_name, + "tenancy_mode": mode, + "fingerprint": _value(**keys), + } + ], + ) + + +async def _prepare(stores: dict[str, Any], network: _Network) -> None: + pid = f"it-{uuid.uuid4().hex[:8]}" + await _beat(stores, pid, cell_name="first", mode="open") + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + assert await cell_prepare.prepare(stores["ch"], args, router, network) == 0 + stores["pid"] = pid + await _beat(stores, pid, mode="recording") + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 20, + "scan_budget": 600, + "dry_run": False, + "ch_allow_user": [settings.ch_user or "default"], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": ROUTER, + } + return argparse.Namespace(**{**fields, **over}) + + +async def _attach(stores: dict[str, Any], network: _Network, **over: Any) -> int: + args = _args(**over) + url = cell_cli.operator_router(args) + assert isinstance(url, httpx.URL) + return await cell_attach.attach(stores["ch"], args, url, network) + + +async def _mode() -> str: + return str(await pg.fetchval("SELECT mode FROM loom.tenancy_mode")) + + +async def _placed() -> set[str]: + import asyncpg + + conn = await asyncpg.connect(settings.router_pg_dsn) + try: + rows = await conn.fetch("SELECT org FROM loom_router.placements WHERE cell = 'first'") + finally: + await conn.close() + return {r["org"] for r in rows} + + +async def _row() -> Any: + return await directory.first_cell_named("first") + + +@pytest.mark.asyncio +async def test_a_recording_first_cell_attaches_to_serving(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + tag = stores["tag"] + await _prepare(stores, network) + capsys.readouterr() + # Orgs a recording cell holds: a tenancy row alone, a Postgres data row, a ClickHouse + # row, and one name no request can carry (skipped and named). + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "signed-in") + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "pg") + await stores["ch"].insert( + "loom.namespaces", + [ + {"org": tag + "ch", "namespace": "n", "owner_email": ""}, + {"org": tag + "bad\x00org", "namespace": "n", "owner_email": ""}, + ], + ) + assert await _attach(stores, network) == 0 + report = json.loads(capsys.readouterr().out) + mine = {tag + "signed-in", tag + "pg", tag + "ch"} + assert report["state"] == "serving" + assert await _mode() == "enforced" + placed = await _placed() + assert {o for o in placed if o.startswith(tag)} == mine + held = {r["org"] for r in await pg.fetch("SELECT org FROM loom.tenancy")} + assert held == placed + assert any("clickhouse loom.namespaces" in k for k in report["skipped"]["malformed"]) + row = await _row() + assert (row.state, row.mode, row.placed) == ("serving", "enforced", len(placed)) + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + # A rerun on `serving` is done. + assert await _attach(stores, network) == 0 + assert json.loads(capsys.readouterr().out)["done"] is True + + +@pytest.mark.asyncio +async def test_a_refusal_after_step_3_leaves_enforced_and_a_rerun_resumes( + cell_stores, monkeypatch, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "a" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + # A process with LOOM_CELL unset appears once the mode is committed: refused at once. + real = cell_attach.Run.durable + + async def durable_then_unset(self: Any) -> int: + xmax = await real(self) + await _beat(stores, f"it-unset-{uuid.uuid4().hex[:6]}", cell_name="") + return xmax + + monkeypatch.setattr(cell_attach.Run, "durable", durable_then_unset) + assert await _attach(stores, network) == 1 + assert "runs with LOOM_CELL unset" in capsys.readouterr().err + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("attaching", "enforced") + # Placed live once enforced; the rerun resumes from the durable read. + monkeypatch.setattr(cell_attach.Run, "durable", real) + for i in list(stores["ids"]): + if i.startswith("it-unset-"): + await _beat(stores, i, cell_name="first", mode="enforced") + await _beat(stores, stores["pid"], mode="enforced") + assert await _attach(stores, network) == 0 + assert org in await _placed() + assert (await _row()).state == "serving" + + +@pytest.mark.asyncio +async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "the test") + assert await _attach(stores, network) == 1 + assert "attach lock: held by pid" in capsys.readouterr().err + assert await _mode() == "recording" + + +@pytest.mark.asyncio +async def test_a_dry_run_writes_nothing(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + capsys.readouterr() + org = stores["tag"] + "dry" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + assert await _attach(stores, network, dry_run=True) == 0 + report = json.loads(capsys.readouterr().out) + assert report["per_source"]["postgres loom.scope_generations"] == { + "count": 1, + "sample": [org], + } + assert await _mode() == "recording" + assert await _placed() == set() + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with pg.dedicated() as conn: + assert any(d.kind == "absent" for d in await cell_triggers.missing(conn)) + + +@pytest.mark.asyncio +async def test_an_org_past_the_limit_refuses_before_any_tenancy_row(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + long = "o" * (cell.ORG_MAX_BYTES + 1) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", long) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('org-ok', '*')") + assert await _attach(stores, network) == 1 + assert f"longer than {cell.ORG_MAX_BYTES} bytes" in capsys.readouterr().err + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert (await _row()).placed == 0 + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_that_pass_unchecked_writers_refuse( + cell_stores, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + db = stores["db"] + role = f"it_role_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + # Stored as first spelled: a fresh session that never set the placeholder. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" SET "Loom.Tenancy_Checked" = \'on\'') + async with pg.dedicated() as conn: + setting = await conn.fetchval( + "SELECT setconfig::text FROM pg_db_role_setting s JOIN pg_roles r " + "ON r.oid = s.setrole WHERE r.rolname = $1", + role, + ) + assert "Loom.Tenancy_Checked" in setting + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [r.subject for r in refusals] == [f"postgres setting for {role} in {db}"] + # Another database's entry does not refuse. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" RESET ALL') + await _admin( + f"ALTER ROLE {role} IN DATABASE postgres SET session_replication_role = 'replica'" + ) + async with pg.dedicated() as conn: + assert await cell_attach_pg.unmarked_refusals(conn, False) == [] + # A role that can insert a tenancy row, a column grant alone, refuses attach. + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + assert await _attach(stores, network) == 1 + err = capsys.readouterr().err + assert f"postgres role {role}: may insert into loom.tenancy" in err + assert await _mode() == "recording" + assert await _attach(stores, network, pg_allow_role=[role], dry_run=True) == 0 + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} IN DATABASE postgres RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_disabled_subscription_refuses(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + name = f"it_sub_{uuid.uuid4().hex[:6]}" + await pg.execute( + f"CREATE SUBSCRIPTION {name} CONNECTION 'host=127.0.0.1 dbname=none' " + "PUBLICATION none WITH (connect = false, enabled = false)" + ) + try: + async with pg.dedicated() as conn: + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [str(r) for r in refusals] == [ + f"postgres subscription {name}: disabled: its apply worker fires no origin " + "trigger; drop it" + ] + finally: + await pg.execute(f"ALTER SUBSCRIPTION {name} SET (slot_name = NONE)") + await pg.execute(f"DROP SUBSCRIPTION {name}") + + +@pytest.mark.asyncio +async def test_step_4_waits_for_a_transaction_that_may_have_read_recording(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as writer, pg.dedicated() as poller: + tx = writer.transaction() + await tx.start() + await writer.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('w', '*')") + # X is read after a later transaction completed, as after the durable read. + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + # A session idle outside a transaction holds nothing: the wait passes. + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + + +@pytest.mark.asyncio +async def test_the_session_checks_and_the_encoding(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as conn: + assert await cell_attach_pg.direct_refusals(conn, "attach") == [] + assert await cell_attach_pg.encoding_refusals(conn) == [] + async with pg.dedicated(marked=False) as conn: + refusals = await cell_attach_pg.direct_refusals(conn, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + f"attach's session loses {pg.TENANCY_MARKER}" + ] + name = f"it_latin_{uuid.uuid4().hex[:6]}" + await _admin( + f"CREATE DATABASE {name} ENCODING 'LATIN1' LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0" + ) + try: + import asyncpg + + from tests.integration._pg_database import with_database + + conn = await asyncpg.connect(with_database(os.environ["LOOM_PG_DSN"], name)) + try: + (refusal,) = await cell_attach_pg.encoding_refusals(conn) + assert "LATIN1, not UTF8" in refusal.reason + finally: + await conn.close() + finally: + await _admin(f"DROP DATABASE {name}") + + +@pytest.mark.asyncio +async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + logged: list[str] = [] + + async def ok() -> bool: + return True + + async with pg.dedicated() as holder, pg.dedicated() as conn: + tx = holder.transaction() + await tx.start() + await holder.execute("SELECT mode FROM loom.tenancy_mode FOR SHARE") + + async def release() -> None: + await asyncio.sleep(3) + await tx.rollback() + + task = asyncio.create_task(release()) + await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await task + assert await _mode() == "enforced" + assert logged == ["the mode's lock is busy; retrying"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "grant,why", + [ + ("GRANT INSERT (org) ON loom.tenancy TO {r}", "may insert into loom.tenancy"), + ("GRANT UPDATE (org) ON loom.tenancy TO {r}", "may update loom.tenancy (org)"), + ("GRANT UPDATE (mode) ON loom.tenancy_mode TO {r}", "may update loom.tenancy_mode (mode)"), + ("GRANT INSERT (mode) ON loom.tenancy_mode TO {r}", "may insert into loom.tenancy_mode"), + ( + "GRANT INSERT, TRUNCATE ON loom.tenancy_mode TO {r}", + "may insert into loom.tenancy_mode", + ), + ( + "GRANT TRIGGER ON loom.scope_generations TO {r}", + "holds TRIGGER on loom.scope_generations", + ), + ("GRANT CREATE ON SCHEMA public TO {r}", "may create in schema public"), + ("GRANT CREATE ON SCHEMA loom TO {r}", "may create in schema loom"), + ( + "GRANT SET ON PARAMETER session_replication_role TO {r}", + "may set session_replication_role", + ), + ( + "SET loom.tenancy_checked = 'off'; " + "GRANT ALTER SYSTEM ON PARAMETER loom.tenancy_checked TO {r}", + "may ALTER SYSTEM SET loom.tenancy_checked", + ), + ("GRANT pg_write_server_files TO {r}", "is a member of pg_write_server_files"), + # A member of the owner role, ADMIN only: it can grant itself the rest. + ("GRANT {owner} TO {r} WITH ADMIN TRUE, INHERIT FALSE, SET FALSE", "owns loom.tenancy"), + ], +) +async def test_each_right_that_makes_an_org_held_is_named( + cell_stores, grant: str, why: str +) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_r_{uuid.uuid4().hex[:6]}" + owner = await pg.fetchval("SELECT current_user") + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) + for sql in grant.format(r=role, owner=owner).split("; "): + await conn.execute(sql) + holders = await cell_attach_pg.privilege_holders(conn) + assert owner in holders.own + assert any(why in w for w in holders.others.get(role, [])), holders.others + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_f_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") + holders = await cell_attach_pg.privilege_holders(conn) + await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") + assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] + finally: + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_repeatable_read_snapshot_from_before_x_holds_step_4(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as reader, pg.dedicated() as poller: + tx = reader.transaction(isolation="repeatable_read") + await tx.start() + await reader.fetchval("SELECT mode FROM loom.tenancy_mode") # its snapshot, no xid + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + + +@pytest.mark.asyncio +async def test_prepare_is_refused_while_attach_holds_the_lock(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "attach") + with pytest.raises(cell_prepare.Refused, match="attach lock: held by pid"): + await cell_prepare.prepare(cell_stores["ch"], args, router, _Network()) + + +@pytest.mark.asyncio +async def test_doctor_names_the_same_holders_settings_and_subscriptions(cell_stores) -> None: + from server.cli import doctor_cell + + await _prepare(cell_stores, _Network()) + role = f"it_d_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await _admin(f"ALTER ROLE {role} SET session_replication_role = 'replica'") + lines = {n: (g, d) for n, g, d in await doctor_cell._writer_reads(())} + assert lines["cell_tenancy_privilege"][0] == "WARN" + assert f"postgres role {role}" in lines["cell_tenancy_privilege"][1] + assert lines["cell_unchecked_writers"][0] == "WARN" + allowed = {n: g for n, g, _d in await doctor_cell._writer_reads((role,))} + assert allowed["cell_tenancy_privilege"] == "PASS" + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "held") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "held" + ) + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + grade, detail = lines["cell_stray_orgs"] + assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail + tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} + assert tight["cell_stray_orgs_unchecked"][0] == "WARN" diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index d1892be846..c78bc40331 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -166,7 +166,7 @@ def walk(node: dict[str, Any]) -> None: ("Index Only Scan", "tenancy_pkey"), ("Index Only Scan", "tenancy_pkey"), ], scans - assert len(await cell_scan._pg_orgs("loom.tenancy", Budget(60))) == 20000 + assert len(await cell_scan.pg_orgs("loom.tenancy", Budget(60))) == 20000 # Every org-keyed table of the schema has the index the scan needs. names, refusals = await cell_scan.list_postgres(own_pg) assert refusals == [] diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py new file mode 100644 index 0000000000..015cebd948 --- /dev/null +++ b/tests/unit/test_cell_attach.py @@ -0,0 +1,682 @@ +"""`loom cell attach` (server/cli/cell_attach.py, cell_attach_beats.py, cell_attach_pg.py) on +fakes: every refusal the heartbeats, the Postgres reads and the router's answers make, the +waits and their budget, the placement batches and the rescan's watch. The real Postgres, +ClickHouse and router behaviour is tests/integration/test_cell_attach.py. CH-free.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import httpx +import pytest + +from server.cli import cell_attach, cell_scan +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal, Refused +from server.cli.cell_scan import Table +from server.core import cell, instances +from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES +from server.settings import settings + +FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} +CURRENT = cell.as_value(FIELDS) +NEXT = cell.as_value({**FIELDS, "oidc_client_id": "next"}) + + +def _value(**keys: Any) -> dict[str, Any]: + return {**CURRENT, "serves": True, "marked": True, "router": True, "router_why": "", **keys} + + +def _beat( + i: str = "p1", + *, + cell_name: str | None = "first", + live: bool = True, + stopped: bool = False, + mode: str = "recording", + value: dict[str, Any] | None = None, + keys_fp: str = "k", +) -> beats.Beat: + return beats.Beat( + instance_id=i, + live=live, + stopped=stopped, + keys_fp=keys_fp, + last_beat="2026-01-01 00:00:00.000", + cell=cell_name, + mode=mode if cell_name is not None else "", + value=(value if value is not None else _value()) if cell_name is not None else None, + ) + + +def _judge(found: list[beats.Beat], stopped_unset: beats.StoppedUnset = "refuse") -> Any: + return beats.judge(found, "first", (CURRENT, NEXT), stopped_unset) + + +def _reasons(verdict: beats.Verdict) -> list[str]: + return [str(r) for r in verdict.refusals] + + +# -- the heartbeats -------------------------------------------------------------------- + + +def test_a_healthy_cell_passes_and_reports_its_keys() -> None: + nxt = {**_value(), **NEXT} + verdict = _judge([_beat("a"), _beat("b", value=nxt)]) + assert (verdict.refusals, verdict.booting, verdict.live, verdict.keys_fps) == ([], [], 2, {"k"}) + + +def test_a_live_process_with_loom_cell_unset_or_another_cell_refuses_at_once() -> None: + verdict = _judge([_beat("a", cell_name=""), _beat("b", cell_name="other")], "wait") + assert _reasons(verdict) == [ + "process a: runs with LOOM_CELL unset: set LOOM_CELL=first on it", + "process b: runs with LOOM_CELL=other: set LOOM_CELL=first on it", + ] + + +def test_a_stopped_unset_process_refuses_or_is_waited_on_as_asked() -> None: + stopped = [ + _beat("a", cell_name="", live=False, stopped=True), + _beat("b", cell_name=None, live=False, stopped=True), + ] + refused = _judge(stopped, "refuse") + assert [r.subject for r in refused.refusals] == ["process a", "process b"] + assert "may still be writing" in refused.refusals[0].reason + waited = _judge(stopped, "wait") + assert (waited.refusals, waited.aging) == ([], ["a", "b"]) + + +def test_a_live_instance_with_no_cell_row_is_booting() -> None: + verdict = _judge([_beat("a", cell_name=None)]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_cell_row_without_the_current_keys_refuses_even_while_booting() -> None: + old = {k: v for k, v in _value().items() if k not in ("serves", "marked", "router")} + verdict = _judge([_beat("a", value=old, mode="")]) + assert _reasons(verdict) == [ + "process a: reports no serves, marked or router key: upgrade it to this release" + ] + + +@pytest.mark.parametrize( + "keys,booting,refused", + [ + ({"marked": False}, [], "lose the tenancy marker"), + ({"marked": None}, ["a"], ""), + ({"router": False, "router_why": "a 404"}, [], "cannot read its router: a 404"), + ({"router": None}, ["a"], ""), + # A workers-only pod places no org: it needs no router credential. + ({"serves": False, "router": False}, [], ""), + ({"serves": False, "router": None}, [], ""), + ], +) +def test_marked_and_router_refuse_or_wait( + keys: dict[str, Any], booting: list[str], refused: str +) -> None: + verdict = _judge([_beat("a", value=_value(**keys))]) + assert verdict.booting == booting + assert (refused in " ".join(_reasons(verdict))) and bool(verdict.refusals) == bool(refused) + + +def test_no_tenancy_mode_yet_is_booting() -> None: + # A pod whose first cell-state read failed: mode '', marked and router null. + verdict = _judge([_beat("a", mode="", value=_value(marked=None, router=None))]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_fingerprint_on_neither_current_nor_next_refuses() -> None: + drifted = {**_value(), **cell.as_value({**FIELDS, "oidc_client_id": "drifted"})} + assert _reasons(_judge([_beat("a", value=drifted)])) == [ + "process a: runs neither loom.cell_self's current nor its next configuration" + ] + + +def test_the_final_refusals() -> None: + scim = {**_value(), **cell.as_value({**FIELDS, "scim": True})} + verdict = beats.judge( + [_beat("a", value=scim)], "first", (cell.as_value({**FIELDS, "scim": True}),), "refuse" + ) + assert [r.reason for r in beats.final_refusals(verdict)] == [ + "SCIM is on: attach with SCIM on needs the per-cell SCIM release" + ] + two = _judge([_beat("a"), _beat("b", keys_fp="other")]) + assert "root-secret fingerprints" in beats.final_refusals(two)[0].reason + assert "no live process" in beats.final_refusals(_judge([]))[0].reason + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock: + c = _Clock() + + async def _sleep(s: float) -> None: + c.now += s + + monkeypatch.setattr(apg, "_sleep", _sleep) + return c + + +@pytest.mark.asyncio +async def test_settle_waits_for_a_booting_process(clock: _Clock) -> None: + reads = [[_beat("a", cell_name=None)], [_beat("a", cell_name=None)], [_beat("a")]] + + async def read() -> list[beats.Beat]: + return reads.pop(0) + + wait = apg.Wait(10, clock) + verdict = await beats.settle(read, _judge, wait) + assert (verdict.live, wait.spent) == (1, 2.0) + + +@pytest.mark.asyncio +async def test_settle_refuses_at_the_end_of_wait_naming_what_it_waited_on(clock: _Clock) -> None: + async def read() -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name="", live=False, stopped=True)] + + with pytest.raises(Refused) as e: + await beats.settle(read, lambda f: _judge(f, "wait"), apg.Wait(3, clock)) + assert [str(r) for r in e.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: stopped with LOOM_CELL unset, not aged out within --wait", + ] + + +@pytest.mark.asyncio +async def test_settle_extends_once_for_a_process_first_seen_booting(clock: _Clock) -> None: + seen = 0 + + async def read() -> list[beats.Beat]: + nonlocal seen + seen += 1 + # A stopped process aging out first; then one booting from the third read (2 s + # spent), done at the seventh, past the first deadline. + if seen < 3: + return [_beat("a"), _beat("x", cell_name="", live=False, stopped=True)] + return [_beat("a"), _beat("b", cell_name=None)] if seen < 7 else [_beat("a"), _beat("b")] + + wait = apg.Wait(3, clock) + verdict = await beats.settle(read, lambda f: _judge(f, "wait"), wait, extend_s=4) + assert (verdict.live, wait.seconds) == (2, 6) + + async def forever() -> list[beats.Beat]: + return [_beat("b", cell_name=None)] + + wait = apg.Wait(2, clock) + with pytest.raises(Refused, match="still booting"): + await beats.settle(forever, _judge, wait, extend_s=1) + # Seen booting from the start: first sight plus the extension is inside the deadline. + assert wait.seconds == 2 + + +@pytest.mark.asyncio +async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: + wait = apg.Wait(2, clock) + clock.now += 100 # a scan's time + assert await wait.pause() and await wait.pause() + assert not await wait.pause() + assert wait.spent == 2.0 + + +# -- the rescan's watch ---------------------------------------------------------------- + + +def test_the_prune_contract_is_an_hour() -> None: + assert instances.PRUNE_AFTER_S == 3600 + + +@pytest.mark.asyncio +async def test_the_watch_refuses_every_unset_process_it_saw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rounds = [ + [ + _beat("a", cell_name=""), + _beat("c"), + _beat("d", cell_name=None, live=False, stopped=True), + ], + # a's rows were swept; c's cell row alone was swept: it was seen with the cell. + [_beat("c", cell_name=None, live=False, stopped=True)], + ] + + async def read_beats(_ch: Any, *, since: str | None = None) -> list[beats.Beat]: + assert since == "2026-01-01 00:00:00.000" + return rounds.pop(0) + + monkeypatch.setattr(beats, "read_beats", read_beats) + clock = _Clock() + watch = beats.Watch(None, "2026-01-01 00:00:00.000", "first", clock) + await watch.read_once() + await watch.read_once() + assert [r.subject for r in watch.refusals()] == ["process a", "process d"] + clock.now += beats.watch_period_s() + 1 + monkeypatch.setattr(beats, "read_beats", read_beats) + rounds.append([]) + await watch.read_once() + assert watch.refusals()[-1].subject == "heartbeats" + + +def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) + assert beats.Watch(None, "", "first").refusals_up_front() + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) + assert beats.Watch(None, "", "first").refusals_up_front() == [] + + +# -- batches, ordering, arguments -------------------------------------------------------- + + +def test_batches_keep_the_count_and_the_byte_bound() -> None: + many = [f"org-{i:05d}" for i in range(ATTACH_BATCH + 1)] + assert [len(b) for b in cell_attach.batches(many, "dep")] == [ATTACH_BATCH, 1] + # Escaping grows a non-ASCII name six times: the encoded body is what counts. + wide = ["é" * 2000 + str(i) for i in range(10)] + out = cell_attach.batches(wide, "dep") + assert len(out) > 1 and sum(map(len, out)) == 10 + for b in out: + body = json.dumps({"deployment_id": "dep", "orgs": b}, separators=(",", ":")) + assert len(body.encode()) <= MAX_BODY_BYTES + + +def test_move_sources_are_scanned_first() -> None: + tables = [Table("loom", n, "MergeTree") for n in ("a", "memory_extraction_queue", "b", "src")] + ordered = cell_scan.order_sources(tables, {"loom.src"}) + assert [t.name for t in ordered] == ["memory_extraction_queue", "src", "a", "b"] + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 10, + "scan_budget": 100, + "dry_run": False, + "ch_allow_user": [], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": "http://127.0.0.1:7900", + } + return argparse.Namespace(**{**fields, **over}) + + +def test_the_arguments(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") + assert cell_attach._args_problem(_args()) == "" + assert "LOOM_CELL" in cell_attach._args_problem(_args(cell="other")) + assert "--scan-budget" in cell_attach._args_problem(_args(scan_budget=0)) + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + assert "Postgres" in cell_attach._args_problem(_args()) + + +def test_an_org_past_the_limit_is_named_with_its_sources() -> None: + found = cell_scan.ScanResult() + found.add("postgres loom.tenancy", "x" * (cell.ORG_MAX_BYTES + 1)) + (refusal,) = cell_attach._too_long(found) + assert refusal.subject.startswith(f"org of {cell.ORG_MAX_BYTES + 1} bytes") + assert "found in postgres loom.tenancy" in refusal.reason + + +# -- the router ------------------------------------------------------------------------ + + +def _router(*responses: httpx.Response) -> cell_attach.Router: + answers = list(responses) + + def handler(_request: httpx.Request) -> httpx.Response: + return answers.pop(0) + + return cell_attach.Router( + httpx.URL("http://127.0.0.1:7900"), "first", httpx.MockTransport(handler) + ) + + +@pytest.mark.asyncio +async def test_the_router_calls_refuse_as_the_router_answers() -> None: + ok = {"cell": "first", "state": "attaching"} + assert await _router(httpx.Response(200, json=ok)).call("") == ok + with pytest.raises(Refused, match="not upgraded"): + await _router(httpx.Response(404, json={"error": "no route"})).call("/attach/place", {}) + with pytest.raises(Refused, match=r"refused /attach/begin \(409\): use loom cell update"): + await _router(httpx.Response(409, json={"error": "use loom cell update"})).call( + "/attach/begin", {} + ) + with pytest.raises(Refused, match="did not answer as the router does"): + await _router(httpx.Response(502, text="bad gateway")).call("") + + +# -- the Postgres reads, on a fake session --------------------------------------------- + + +@dataclass +class FakeConn: + """Answers by the first needle the SQL holds; a callable answer is called with the + arguments; an exception answer is raised.""" + + routes: list[tuple[str, Any]] = field(default_factory=list) + executed: list[str] = field(default_factory=list) + + def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else answer + raise AssertionError(f"unrouted: {sql[:80]}") + + async def fetch(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchrow(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchval(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def execute(self, sql: str, *args: Any) -> str: + self.executed.append(sql) + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else str(answer) + return "OK" + + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + + return _Tx() + + +def _seq(*values: Any) -> Callable[..., Any]: + pending = list(values) + return lambda *_a: pending.pop(0) + + +@pytest.mark.asyncio +async def test_the_session_checks_refuse_a_pooler() -> None: + moved = FakeConn( + [ + ("pg_backend_pid() AS pid", _seq((1, "t0"), (2, "t1"))), + ("current_setting($1, true)", None), + ] + ) + refusals = await apg.direct_refusals(moved, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + "attach's session moved between backends", + "attach's session loses loom.tenancy_checked", + ] + direct = FakeConn([("pg_backend_pid() AS pid", (1, "t0")), ("current_setting($1, true)", "on")]) + assert await apg.direct_refusals(direct, "attach") == [] + assert "RESET ALL" in direct.executed + + +@pytest.mark.asyncio +async def test_the_lock_refuses_when_held_or_not_on_this_backend() -> None: + holder = [{"pid": 7, "application_name": "loom", "client": "10.0.0.1", "backend_start": "t"}] + held = FakeConn([("pg_try_advisory_lock", False), ("FROM pg_locks l LEFT JOIN", holder)]) + with pytest.raises(Refused, match="held by pid 7 \\(loom, from 10.0.0.1"): + await apg.take_lock(held, "attach") + pooled = FakeConn([("pg_try_advisory_lock", True), ("SELECT EXISTS", False)]) + with pytest.raises(Refused, match="a pooler"): + await apg.take_lock(pooled, "attach") + assert any("tcp_keepalives_idle" in s for s in pooled.executed) + + +@pytest.mark.asyncio +async def test_a_database_not_utf8_refuses() -> None: + assert await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "UTF8")])) == [] + (r,) = await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "SQL_ASCII")])) + assert "SQL_ASCII, not UTF8" in r.reason + + +def test_privilege_holders_outside_the_allowed_refuse() -> None: + holders = apg.Holders( + own=["loom"], + others={"ci": ["may insert into loom.tenancy"], "servers": ["owns loom.memory_current"]}, + ) + assert [r.subject for r in apg.privilege_refusals(holders, ["servers"])] == ["postgres role ci"] + assert apg.privilege_refusals(holders, ["ci", "servers"]) == [] + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_refuse() -> None: + conn = FakeConn( + [ + ( + "pg_db_role_setting", + [{"role": "ci", "db": "every database", "cfg": "Loom.Tenancy_Checked=on"}], + ), + ("pg_subscription", [{"subname": "mirror", "subenabled": False}]), + ] + ) + refusals = await apg.unmarked_refusals(conn, True) + assert [r.subject for r in refusals] == [ + "postgres unmarked session", + "postgres setting for ci in every database", + "postgres subscription mirror", + ] + + +@pytest.mark.asyncio +async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: + conn = FakeConn( + [ + ("SELECT current_user", "attacher"), + ("WHERE NOT held", [{"need": "membership in pg_read_all_stats"}]), + ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), + ("max_prepared_transactions", 5), + ] + ) + refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + assert [str(r) for r in refusals] == [ + "postgres role attacher: attach's role lacks membership in pg_read_all_stats", + "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " + "session, so step 4 could not wait for it; set it to 0", + ] + + +def _row(**over: Any) -> dict[str, Any]: + return { + "pid": 9, + "backend_type": "client backend", + "application_name": "ci", + "xact_start": "t", + "ours": True, + "xid_old": False, + "xmin_old": False, + "hidden": False, + **over, + } + + +@pytest.mark.asyncio +async def test_step_4_refuses_what_it_cannot_wait_for(clock: _Clock) -> None: + base = [("max_prepared_transactions", 0), ("pg_prepared_xacts", 0)] + hidden = FakeConn([*base, ("pg_stat_activity s", [_row(hidden=True)])]) + with pytest.raises(Refused, match="hidden from attach's role"): + await apg.wait_transactions(hidden, 100, apg.Wait(5, clock)) + prepared = FakeConn([("max_prepared_transactions", 0), ("pg_prepared_xacts", 1)]) + with pytest.raises(Refused, match="prepared transaction"): + await apg.wait_transactions(prepared, 100, apg.Wait(5, clock)) + held = FakeConn( + [ + *base, + ( + "pg_stat_activity s", + [_row(xmin_old=True), _row(pid=10, ours=False, xid_old=True), _row(pid=11)], + ), + ] + ) + with pytest.raises(Refused) as e: + await apg.wait_transactions(held, 100, apg.Wait(2, clock)) + assert [r.reason for r in e.value.refusals] == [ + "may have read `recording`: still open", + "another database's transaction, holding the snapshot horizon", + ] + done = FakeConn([*base, ("pg_stat_activity s", [_row(pid=10, ours=False, xid_old=True)])]) + await apg.wait_transactions(done, 100, apg.Wait(0, clock)) + + +class _Timeout(Exception): + sqlstate = "55P03" + + +@pytest.mark.asyncio +async def test_the_mode_commit_retries_a_busy_lock_and_ends_an_earlier_run(clock: _Clock) -> None: + tries = _seq(_Timeout(), "UPDATE 1") + + def lock(*_a: Any) -> str: + answer = tries() + if isinstance(answer, Exception): + raise answer + return "LOCK TABLE" + + holders = _seq( + [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}], [] + ) + conn = FakeConn( + [ + ("LOCK TABLE", lock), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", holders), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + confirmed: list[bool] = [] + + async def confirm() -> None: + confirmed.append(True) + + async def ready() -> bool: + return True + + logged: list[str] = [] + await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + assert logged == ["the mode's lock is busy; retrying"] + # Before each try and the one termination. + assert len(confirmed) == 3 + + +@pytest.mark.asyncio +async def test_a_superusers_mode_transaction_is_named_not_ended() -> None: + conn = FakeConn( + [ + ( + "l.mode = 'ExclusiveLock'", + [{"pid": 5, "application_name": "psql", "xact_start": "t", "rolsuper": True}], + ), + ("rolsuper FROM pg_roles", False), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused, match="a superuser must end it"): + await apg.end_mode_transactions(conn, confirm) + + +@pytest.mark.asyncio +async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( + clock: _Clock, +) -> None: + conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + + async def confirm() -> None: + pass + + async def booting() -> bool: + return False + + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + assert not any("LOCK TABLE" in s for s in conn.executed) + + +@pytest.mark.asyncio +async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) + + async def read(mode: str, xid: int) -> apg.Durable: + return apg.Durable(mode=mode, first=(True, "first"), xid=xid) + + monkeypatch.setattr(apg, "durable_read", lambda: read("recording", 50)) + with pytest.raises(Refused, match="did not land"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 100)) + with pytest.raises(Refused, match="is not above 100"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 99)) + assert await run.durable() == 100 + + async def waiting() -> str: + return "waiting" + + monkeypatch.setattr(pg_profile, "standby_now", waiting) + with pytest.raises(Refused, match="not synchronous before"): + await run.durable() + + +@pytest.mark.asyncio +async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + state = ["enforced"] + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return state[0], (True, "first") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "read_mode", read_mode) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + assert await run.done() + state[0] = "recording" + with pytest.raises(Refused, match="run loom cell restore"): + await run.done() + + +def test_refusal_text() -> None: + assert str(Refusal("a", "b")) == "a: b" diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 3f670d4014..decb365521 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -695,7 +695,7 @@ def test_doctor_runs_the_cell_checks_only_with_loom_cell( monkeypatch.setattr(doctor, "_doctor_scim_token_check", lambda _s: ("scim_token", "PASS", "")) called: list[bool] = [] monkeypatch.setattr( - doctor_cell, "cell_checks", lambda: called.append(True) or [("cell_state", "PASS", "")] + doctor_cell, "cell_checks", lambda *_: called.append(True) or [("cell_state", "PASS", "")] ) names = [n for n, _g, _d in doctor._run_live_checks()] assert called == ([True] if runs else []) and ("cell_state" in names) == runs diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index e9ea657df6..c978eb94e2 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import contextlib import json from datetime import UTC, datetime from typing import Any @@ -29,6 +30,8 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "storage_authority", "postgres") monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). + monkeypatch.setattr(cell_prepare, "attach_lock", contextlib.nullcontext) def _args(**over: Any) -> argparse.Namespace: diff --git a/tests/unit/test_doctor_client_checks.py b/tests/unit/test_doctor_client_checks.py index 554220a424..a2e1b0bd01 100644 --- a/tests/unit/test_doctor_client_checks.py +++ b/tests/unit/test_doctor_client_checks.py @@ -572,7 +572,7 @@ def test_doctor_client_flag_appends_the_lines_and_json_carries_them( ) -> None: from server.cli import doctor - monkeypatch.setattr(doctor, "_run_doctor_checks", lambda: [("python", "PASS", "3.12")]) + monkeypatch.setattr(doctor, "_run_doctor_checks", lambda *_: [("python", "PASS", "3.12")]) monkeypatch.setattr(doctor, "_doctor_problems", lambda: []) monkeypatch.setattr( doctor, diff --git a/tests/unit/test_doctor_client_client_only.py b/tests/unit/test_doctor_client_client_only.py index 60ceb323d6..712c10d7cd 100644 --- a/tests/unit/test_doctor_client_client_only.py +++ b/tests/unit/test_doctor_client_client_only.py @@ -56,7 +56,7 @@ def test_clickhouse_line_names_the_probed_host_on_failure(monkeypatch: pytest.Mo assert detail.startswith("https://ch.example.com:8443 → TimeoutError") -def _live() -> list[Check]: +def _live(*_roles: Any) -> list[Check]: return [ ("python_version", "PASS", "3.12"), ("clickhouse_reachable", "FAIL", "https://ch.example.com:8443 → TimeoutError: timed out"), From 6cf53b3c58ca49427c3e7ecfe4a8e5d33823ec62 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:19:41 +0000 Subject: [PATCH 02/25] Attach's runbook, the cells.md first-attach steps and the trigger-cost gate measured end to end docs/operations/cell-attach.md is the operator's procedure: the preconditions attach checks and those it relies on, the flags, the point of no return at the mode commit, the orgs attach does not place and the manual removal of an org past the length limit, and doctor's reads after serving. cells.md describes the first attach as it runs (the lock, the refusals, the trigger pass, the mode commit and its durable read, step 4's wait, rescan and final checks), with its stated limits and failure rows. upgrades.md step 18 rotates the ClickHouse writer credential after the LOOM_CELL rollout. bench/cell_trigger_gate.py measures memory.set and a delivery-sized insert through an attached cell with the org triggers on and off; results.md records a local run. Attach compares the final root-secret fingerprint with the one its begin wrote, and the nonce cleanup's message names no single command. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 117 ++++++++++++ docs/architecture/cells.md | 290 +++++++++++++++++++---------- docs/benchmarks/results.md | 31 ++- docs/operations/cell-attach.md | 148 +++++++++++++++ docs/operations/postgres.md | 4 + docs/operations/upgrades.md | 8 + docs/reference/cli.md | 3 +- llms.txt | 1 + server/cli/cell_attach.py | 10 +- server/cli/cell_prepare.py | 2 +- tests/unit/snapshots/cli_help.json | 12 +- tests/unit/snapshots/cli_tree.json | 261 +++++++++++++++++++++++++- tests/unit/test_cell_attach.py | 17 ++ 13 files changed, 791 insertions(+), 113 deletions(-) create mode 100644 bench/cell_trigger_gate.py create mode 100644 docs/operations/cell-attach.md diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py new file mode 100644 index 0000000000..e90fbb32fa --- /dev/null +++ b/bench/cell_trigger_gate.py @@ -0,0 +1,117 @@ +"""The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server +of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an +unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` +guard stays). Run it against an attached cell's server, which must not serve anything else, +with a writer token for a placed org; it drops and recreates the cell's org triggers, so +never against a deployment in use. Needs no provider. + + PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ + --base-url http://127.0.0.1:7932 --pg-dsn \\ + --token --org --namespace +""" + +from __future__ import annotations + +import argparse +import asyncio +import statistics +import time +import uuid + +import asyncpg +import httpx + +from server.core import cell_triggers + +CALLS, WARMUP, ROUNDS, INSERTS = 400, 50, 3, 20 + + +async def _triggers(dsn: str, on: bool) -> None: + """Create or drop the org triggers, and check that what was asked holds: every guarded + table carries both triggers enabled at origin, or none does.""" + conn = await asyncpg.connect(dsn) + try: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + for table in guarded: + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + enabled = await conn.fetchval( + "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", + list(cell_triggers.ORG_TRIGGERS), + ) + finally: + await conn.close() + expected = 2 * len(guarded) if on else 0 + if enabled != expected: + raise SystemExit(f"{enabled} org triggers enabled, expected {expected}") + + +async def _memory_set(args: argparse.Namespace) -> tuple[float, float]: + """p50 and p99 in ms of CALLS sequential `memory.set` calls, each a new key.""" + lat = [] + headers = {"Authorization": f"Bearer {args.token}"} + async with httpx.AsyncClient(timeout=30, headers=headers) as client: + for i in range(CALLS + WARMUP): + body = { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": f"gate-{uuid.uuid4().hex[:10]}", + "value": f"trigger gate {i}", + } + t = time.perf_counter() + (await client.post(f"{args.base_url}/v1/memory.set", json=body)).raise_for_status() + if i >= WARMUP: + lat.append((time.perf_counter() - t) * 1000) + lat.sort() + return lat[len(lat) // 2], lat[int(0.99 * (len(lat) - 1))] + + +async def _batch(args: argparse.Namespace, marked: bool) -> float: + """The median in ms of INSERTS 1,000-row inserts of the held org, in one statement each.""" + settings = {"loom.tenancy_checked": "on"} if marked else {} + conn = await asyncpg.connect(args.pg_dsn, server_settings=settings) + try: + times = [] + for _ in range(INSERTS): + t = time.perf_counter() + await conn.execute( + "INSERT INTO loom.scope_generations (org, scope) " + "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + args.org, + uuid.uuid4().hex[:8], + ) + times.append((time.perf_counter() - t) * 1000) + return statistics.median(times) + finally: + await conn.close() + + +async def main() -> None: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--base-url", required=True) + p.add_argument("--pg-dsn", required=True) + p.add_argument("--token", required=True) + p.add_argument("--org", required=True) + p.add_argument("--namespace", required=True) + args = p.parse_args() + try: + for _ in range(ROUNDS): + for on in (True, False): + await _triggers(args.pg_dsn, on) + p50, p99 = await _memory_set(args) + marked, unmarked = await _batch(args, True), await _batch(args, False) + print( + f"triggers {'on ' if on else 'off'}: memory.set p50 {p50:.1f} ms " + f"p99 {p99:.1f} ms; 1,000-row insert {marked:.1f} ms marked, " + f"{unmarked:.1f} ms unmarked", + flush=True, + ) + finally: + await _triggers(args.pg_dsn, True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..96580ec7d2 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -389,7 +389,8 @@ open meanwhile, as on a deployment never prepared, since the cell's Postgres, wh holds its state, cannot tell the two apart (ClickHouse carries no cell state: adding it would make the write path read a second store for tenancy) and such a process never reads the directory; where every process has `LOOM_CELL` unset, `loom doctor`'s naming -of it (which reads the directory, and ships with attach) is the only signal. A +of it (which reads the directory, and ships with the router's read-only scope) is the only +signal. A `(true, …)` row with `recording` and the own row: the first cell recording. Both present with the own row otherwise: `enforced`, a later cell's mode reading as `enforced` whatever its row says (its triggers, mode cache and CLI treat it so, and @@ -676,20 +677,18 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after any error in that -transaction it ends that session's backend (`pg_terminate_backend`, the backend named by -the pid and `backend_start` attach captured when it connected) and waits until it has -left `pg_stat_activity`, then reads the mode back before it retries or reports; a mode -read back as `enforced` is written again in a fresh transaction, whose commit waits for -the standby, before attach goes on, since a commit ended while it waited is on the -primary alone. The attach command fixes those timeouts, and its tests cover each of -these cases. The mode cache reads the mode with a plain `SELECT`, which takes no row -lock and never waits on attach. An insert that finds `enforced` (a process whose cache +server-side timeouts, then writes the mode and commits; after a failed try it ends the +mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_mode` in +this database (a pid captured at connect is wrong behind a connection path that swaps +backends), and waits until the lock clears. Whether the commit landed is decided by a +durable read of the mode, whose commit waits for the standby, since a commit ended while +it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never +waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in -`enforced`, and a CLI command refuses as in `enforced`, naming the org. A process whose -cache still reads `enforced` after step 3 or step 4 returns the cell to `recording` places -through the router's gate until its cache expires. -`loom doctor` lists any tenancy row the directory does not place. +`enforced`, and a CLI command refuses as in `enforced`, naming the org. A first attach +never lowers the mode once it commits `enforced`. `loom doctor`, with the router's +read-only scope, lists any tenancy row the directory does not place. - **It runs after verification, before anything is written for the org.** The op gate (`gates.authorize`) checks the org the transport resolves (`gates.resolve_org`, with @@ -983,51 +982,126 @@ refuses on any failure, naming it: revocation prefix, so a person the identity provider provisions later is admitted, since it will never deprovision a person it does not count as assigned. -The first attach turns the existing deployment into the first cell, in order: - -1. check the mode is `recording`, as prepare set it: every verified request has written - its org's tenancy row if absent while the directory answered that the cell is - `attaching`; -2. in one directory transaction, write `attach_started_at`, the cell's issuers, patterns - and fingerprint (both values of `loom.cell_self`), and the placements, then the - tenancy rows, of every org the cell holds, from one snapshot, straight into the - directory (no other cell exists yet): its configured orgs (of both values), the - distinct `org` of every table of both schemas (generated from the schema, as - erasure's table classification is), and the orgs of catalogued bucket objects; never - `''`, `'-'` or `'*'`; -3. set `enforced` as above, within `--wait`, which the wait below then shares, starting - no try that could commit the mode with less left than that wait's minimum (the mode - cache's lifetime plus the longest request deadline): attach then refuses, reporting - the mode it reads back (`recording`, and a rerun resumes from step 1), or the mode as - unknown when it cannot read it, and a rerun then resumes by the mode it reads; then - wait, within what remains of `--wait` (default 10 minutes, then refuse and set - `recording` back), until every live process reports `enforced` and then the mode - cache's lifetime plus the longest request deadline have passed; then run the SCIM - precondition again (`--revoke-unprovisioned` again if given), for accounts made - before `enforced` took hold; if it refuses, the cell stays `enforced` and - `attaching`, and running attach again resumes from the wait, as a resumed attach does - (below); -4. wait, within what remains of `--wait`, until no transaction that could have read - `recording` is still open, judged from a Postgres snapshot attach takes once it has - read `enforced` back in this run (not from a clock): a transaction on the cell's - database that started before it, from any client, background worker or logical - replication connection, read-only ones included, a writing transaction on another - database of the same server that did (it holds back every snapshot's horizon, and is - named as such), a prepared transaction (which refuses at once), or one whose state - attach cannot read counts as open, and Postgres's own maintenance processes, physical - replication connections (every standby's and a base backup's, which run no SQL) and - attach's own connections do not. Past `--wait` attach refuses, naming what it waited - on, and leaves the cell `enforced`, so a rerun resumes from step 3's wait. The attach - command fixes the exact query, and its tests cover each kind of transaction. Once the - wait passes, place the tenancy rows written since the snapshot and every org step 2's - sources yield when read again (the distinct `org` of every table of both schemas and - the orgs of catalogued bucket objects; a row or an object an unchecked writer, such - as an older CLI or a worker, wrote in `recording` after the snapshot is placed too), - check again that every live process reports `enforced` (refusing and returning to - `recording` as step 3 does if one does not), and mark the cell `serving`. The - directory records the highest mode attach has raised or tried to raise, which - `loom cell restore` writes back only upward: a cell whose step 3 was refused at the - lock is restored `enforced`, and its resumed attach then runs from step 3's wait. +The first attach (`loom cell attach`, its procedure on +[attaching the first cell](../operations/cell-attach.md)) turns the existing deployment +into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at +`attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and +`loom cell update` take the same lock), after checking that the session is one direct +backend that keeps the tenancy marker: a lock left on a pooler's server connection would +refuse every later run, and attach's tenancy rows would be refused as an unchecked +writer's. A lock found lost between steps is taken again (refusing when another run holds +it), and the cell's state is read again. The lock does not make a run safe, since every +step is idempotent and the mode only rises; it keeps a second run's scans from holding the +snapshot horizon that step 4 waits on. + +It refuses, before any write, naming each: + +- stores other than the registered ones: the Postgres deployment id, ClickHouse's + `loom.storage_authority` and the shared host's `/v1/health/deployment` each name the + directory row's deployment id, `loom.cell_first` reads `(true, )`, the row is the + first cell's, `attaching`, with no other directory row; +- a database encoding other than UTF8 (prepare refuses the same), so a tenancy row + within the length limit is always an org a request can name; +- a synchronous standby not attached now, a durability profile that would not refuse + without one, or a role that may not write the durable read's logical message; a role + of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with + `max_prepared_transactions` above 0; +- **the tenancy-privilege check**: a login role, other than attach's own (superusers, + and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and + those named with `--pg-allow-role`, that holds or reaches through any membership a right + to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or + `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, + `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system + one (a function there can shadow one the servers call unqualified on their marked + session), `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` on + `loom.tenancy_checked`, membership in `pg_write_server_files` or + `pg_execute_server_program`, or ownership of a `loom` table, of the trigger functions or + of `loom.cell_first_immutable()`, the database or a schema; +- **the unmarked-session check**: a session that sends no marker reads one, or runs as + `replica`; a `pg_db_role_setting` entry of `loom.tenancy_checked` or + `session_replication_role` for this database or every one, matched case-insensitively; + any logical-replication subscription into this database, enabled or not (its apply + worker fires no origin trigger); +- the heartbeats (both tables, stopped rows included for the widened window, the live + window plus a stopping pod's drain and retirement): a live process with `LOOM_CELL` + unset or another cell, one without the heartbeat's keys, `marked` false, `router` false + where `serves` is true, a fingerprint on neither `loom.cell_self` value, several + `keys_fp` values, or none live; a booting process (no cell row yet, no mode, `marked` + or `router` null) is waited on within `--wait`; in `recording` a stopped process with + `LOOM_CELL` unset refuses too; +- the ClickHouse writer check and the scan's table list (`server/cli/cell_writers.py`, + `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user + reads. + +Then, in order: + +1. `attach/begin`, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, + through `attach/enforced` again, to the durable read. +3. Create the triggers itself, on every run, before step 2's scan and again just before the + mode commit (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads + every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an + `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with + the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only + org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such + orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row + is written; a name no request can carry is skipped and named. Then the tenancy rows of + the orgs found, on a marked session, and no placement: unscoped callers are answered + from the tenancy rows, and placing now would move the point of no return earlier. +5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of + `--scan-budget`, since step 4 reads the same tables again. Each refuses in + `recording`, which can still be backed out. +6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on + `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock + (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database + alone) and ended. From the commit on the mode only rises: every later refusal leaves + the cell `enforced` and `attaching`, and a rerun resumes from the durable read. +7. The durable read, in every run that reads `enforced`: the mode and the `first` row read + in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. + Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED + writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a + snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is + assigned after the mode commit. +8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends + on its cached mode (a request records a new org through `tenancy._RECORD`, which reads + the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling + one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A + prepared transaction, or a row of the cell's database whose state is hidden, refuses + at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the + heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); + the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery + tables move rows to ClickHouse), and the queue and drain tables and the tables a view + reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, + in `attach/place` batches; the writer check again; and the final checks, under a floor + of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): + the heartbeats, read through the rescan every half of the time the leader keeps a + stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` + unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` + equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 +hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would +be placed and skipped and the scan's time, and writes nothing; the router's checks at +`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is +done (exit 0), anything else refuses, naming `loom cell restore`. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1046,13 +1120,18 @@ cell with SCIM on admits no one until its first provisioning cycle has run. A ce nearing its stores' limits is closed to new orgs: its placement calls answer 503, so its issuers' new orgs wait for capacity rather than spill elsewhere. -A resumed first attach, for a cell whose directory row has `attach_started_at`, reruns -every refusal of the list above that applies to the first cell. It then continues by the -cell's mode: in `recording` from step 1; in `enforced`, from step 3's wait (every live -process reports `enforced`, then the cache's lifetime and the longest request deadline -pass) and then its SCIM precondition, so the SCIM check that ends an attach stopped -after setting `enforced`, whatever stopped it, always runs after `enforced` has taken -hold. +A resumed first attach reruns every refusal and continues by the directory row and the +cell's mode: in `recording` from the scan, in `enforced` from the durable read; whenever +the cell reads `enforced` it posts `attach/enforced` again, so a directory restored below +the cell's mode is raised. + +The checks of the stores' writers are point-in-time: a grant, a role setting, a +subscription, a ClickHouse writer or object added after `serving` passes them, as does a +configuration file attach cannot read (`postgresql.conf`, `postgresql.auto.conf`, a +standby's own) or a role that runs code as the server's OS user. `loom doctor` runs the +tenancy-privilege and unmarked-session reads again on every run and warns on what it finds +(`cell_tenancy_privilege`, `cell_unchecked_writers`); the runbook has the operator confirm +the rest. ## What each cell runs @@ -1102,9 +1181,9 @@ closed: every cell keeps serving the orgs it holds, routing continues from each replica's signed snapshot of its last cache (written to shared storage on every directory change), and the placements, lookups of uncached orgs, `loom cell` commands that read or write the directory and the recording cells' state reads answer 503 or -refuse until the directory is back, while `loom doctor` reports its directory checks as -not run and, with `LOOM_CELL` set, exits non-zero; rebuilding a lost directory from the -cells is a design of its own. +refuse until the directory is back, while `loom doctor`, with the router's read-only +scope, reports its directory checks as not run and, with `LOOM_CELL` set, exits +non-zero; rebuilding a lost directory from the cells is a design of its own. Each cell keeps, in its own Postgres, a copy of its directory row's configuration fingerprint (`loom.cell_self`), which its processes compare their configuration with and @@ -1200,30 +1279,36 @@ the directory row. ## Failure modes -| Failure | What happens | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | -| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | -| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | -| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | -| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | -| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | -| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | -| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | -| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | -| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | -| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | -| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | -| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | -| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | -| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | -| A recording insert held open during first attach | attach stops trying once too little of `--wait` is left for the wait, reporting the mode it reads back; a rerun resumes | -| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | -| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | -| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach | -| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| Failure | What happens | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | +| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | +| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | +| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | +| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | +| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | +| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | +| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | +| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | +| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | +| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | +| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | +| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | +| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | +| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | +| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | +| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | +| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | +| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | +| A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | +| `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | +| A process that changes between attach's last check and the mode commit, or an HA manager that empties `synchronous_standby_names` between the checks around the durable read | not seen by attach: a window of seconds | +| A cell process with a longer `LOOM_INSTANCE_HEARTBEAT_SECONDS` than attach's | it can fall outside attach's live window while live; the runbook requires one value | ## Rolling upgrades @@ -1306,11 +1391,16 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). `loom doctor` names), with their functions in `schemas/pg_migrations/011_tenancy_triggers.sql` and `012_tenancy_guard.sql`; `server/cli/doctor_cell.py`: `loom doctor`'s cell checks; -- `server/cli/cell_scan.py`: the first attach's org scan, a library no command calls: the - tables of both stores with an `org` column, judged by engine and against erasure's - classification, every host of a cluster listing the same, and the distinct orgs of each - table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the - users, user directories, objects and replicas outside the operator's allow lists); +- `server/cli/cell_attach.py`: `loom cell attach`; `server/cli/cell_attach_pg.py`: its + Postgres reads and writes (the session checks, the attach lock, the tenancy-privilege + and unmarked-session checks doctor shares, the trigger pass, the mode commit, the + durable read, step 4's wait); `server/cli/cell_attach_beats.py`: its judgment of the + heartbeats and the rescan's watch; +- `server/cli/cell_scan.py`: the first attach's org scan: the tables of both stores with + an `org` column, judged by engine and against erasure's classification, every host of a + cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; + `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, + objects and replicas outside the operator's allow lists); `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; @@ -1386,14 +1476,14 @@ An integration test with two cells and a router in CI's Postgres job: CLI censuses cover the rest); - the first attach places every org with a row in any table, an org first seen during the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's snapshot and `enforced`, a ClickHouse row an older CLI wrote then and a + step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer request, passes its health probe, answers the probe route and runs no worker on an attached cell and on a later cell between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included after step 3 or step 4 returned the cell to it; a CLI command + `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token revocation) runs, also with Postgres down on a deployment never prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 07367d3688..81e099d3c8 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1220,8 +1220,35 @@ a row for the state read and the `loom.tenancy` lookup. Loom's own writes are ma this predicts the ship gate (a `memory.set` p99 rise of at most 10 ms, a delivery pass at most 10% slower) holds by a wide margin; the cost lands on scripts and older builds writing a cell without the check. This measures the triggers on the statements, not a -whole `memory.set` through a server on a cell: the gate itself is measured end to end -with attach. +whole `memory.set` through a server on a cell; the end-to-end gate follows. + +`bench/cell_trigger_gate.py` measures the gate end to end, locally: a first cell attached +by `loom cell attach` (two server processes, a router, Postgres 16 with a synchronous +standby and ClickHouse in Docker on a shared host under other load), 400 sequential +`memory.set` calls of a placed org through one server, and a delivery-sized 1,000-row insert +of the held org from a marked and an unmarked session, the org triggers on and dropped in +turn, three rounds each; the harness checks the triggers it asked for are enabled: + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url \ + --pg-dsn --token --org --namespace +``` + +| Local, three rounds | Triggers on | Triggers off | +| ----------------------------------- | ---------------------- | ---------------------- | +| `memory.set` p50 | 154.4, 99.3, 98.4 ms | 91.9, 101.2, 101.6 ms | +| `memory.set` p99 | 494.0, 291.7, 257.6 ms | 279.2, 244.7, 233.3 ms | +| 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | +| 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | + +The marked batch, which is what a delivery pass of a cell's own processes writes, shows no +cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. +The `memory.set` figures move by more between rounds of one configuration than any trigger +cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms +here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no +consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the +host's noise, and the statement-level figures above bound the marked cost at under a +microsecond a row. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md new file mode 100644 index 0000000000..12a0cbe247 --- /dev/null +++ b/docs/operations/cell-attach.md @@ -0,0 +1,148 @@ +# Attaching the first cell + +`loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, +directory row `attaching`) into a `serving` one: it places every org the cell holds in the +router's directory, raises the cell's mode to `enforced` and creates its org triggers. This +page is the operator's procedure; the design is +[cells](../architecture/cells.md#attaching-a-cell). + +## Before attach + +Run attach at low load, as one run at a time, and not during a rollout, a provider +transition, a logical backup, a long erasure or a migration: each holds a transaction or a +snapshot long enough for step 4's wait to run out of `--wait`. Steps 2 and 4 run index skip +scans on the Postgres primary (each statement holds vacuum's horizon while it runs) and +read the `org` column of every ClickHouse table, which competes with serving load. + +Every item below is a precondition attach checks, or one it cannot check and relies on: + +1. Every router replica runs this release; every process and every CLI and CI image of + the cell runs it too ([upgrades](upgrades.md#upgrade-procedure), steps 16 and 17). +2. `LOOM_CELL` is set on every process of the cell. Every CLI and CI configuration that + writes for an org has `LOOM_CELL` set without `LOOM_CELL_ROUTER_TOKEN`: in `recording` + it then refuses only new orgs, so nothing has to change at the mode commit. With the + token removed, revoke the `INSERT` on `loom.tenancy` and the `UPDATE (since)` on + `loom.tenancy_mode` the CI role held for recording new orgs. +3. The ClickHouse writer credential is rotated after that rollout + ([upgrades](upgrades.md#upgrade-procedure), step 18). +4. No fingerprinted setting has changed since prepare, and none changes until attach ends: + a change gates every process against `loom.cell_self`. +5. `LOOM_SERVES_TRAFFIC=false` is set only on pods no load balancer routes to: attach + exempts them from the router check. +6. The cell's processes and the attach CLI share one `LOOM_INSTANCE_HEARTBEAT_SECONDS`: + attach judges which processes are live by its own value. +7. `LOOM_PG_DSN` points straight at Postgres, with no pooler of any mode, for the servers + and for attach: attach refuses a session that moves between backends or loses the + tenancy marker, and refuses a server whose heartbeat reports `marked` false. +8. SCIM is off: attach refuses with it on. +9. On a ClickHouse cluster of several hosts, attach's user exists on every host, and the + cluster has an interserver `` or a `remote_servers` user equal to attach's. + One replica that cannot sync blocks attach (`SYSTEM SYNC REPLICA … ON CLUSTER`): an + availability cost, not a safety one. +10. Only Loom's own roles and superusers hold the rights attach's tenancy-privilege check + counts (inserting a tenancy row, changing the mode, creating in a schema, triggers on + `loom` tables, owning them or their trigger functions, `session_replication_role`, + `ALTER SYSTEM` of `loom.tenancy_checked`, `pg_write_server_files`, + `pg_execute_server_program`), and no role that holds one, or owns what it counts, owns + a `SECURITY DEFINER` function another role can execute that does any of that. Attach + checks the catalog; it cannot read a function's body for intent, or a role that runs + code as the server's OS user. +11. `postgresql.conf`, `postgresql.auto.conf` and any pending `ALTER SYSTEM` entry, on the + primary and on every standby, set neither `loom.tenancy_checked` nor + `session_replication_role`. Attach sees a live entry on the primary (its unmarked + session reads it), never a standby's files. +12. Postgres has a synchronous standby attached (`synchronous_standby_names` set) and + `max_prepared_transactions = 0`; the database encoding is UTF8. +13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. + +Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` on schema +`loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, +`loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on +`loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role +of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the +writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT +QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings +profile or quota capping the rows or bytes it reads, and the insert into +`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the +bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. + +## Run it + +```bash +loom cell attach --cell --dry-run \ + --ch-allow-user --pg-allow-role +loom cell attach --cell \ + --ch-allow-user --pg-allow-role +``` + +`--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs +per source table, the ones it would skip, and whether the scan fit in half of +`--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object +and replica that may write the configured databases with `--ch-allow-user`, +`--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every +Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each +list is audited by the router. `--wait` (default 600 s) bounds the waits together; +`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of +it, since step 4 reads the same tables again. + +A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a +rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it +is done. + +## The point of no return + +Step 3's mode commit is one-way. Before it, a refused attach leaves the cell `recording`, +its tenancy rows and its triggers in place (they pass every row in `recording`), and the +cell can be backed out (the first-cell branch of +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated)): +`loom cell remove` serves only a first cell that never left `recording`. After it, scoped +principals place their orgs live within seconds, every refusal leaves the cell `enforced` +and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unsetting +`LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` +always goes forward. + +From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account +routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an +ordinary mint scoped to its org places the org on its first request. A direct call of the +router's operator place route, or a live placement with the cell's credential, cannot read +the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's +credential is the servers' alone. + +## Orgs attach does not place + +- **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is + written. One first found in step 4 is skipped and named. Remove such an org by hand + until the stray-org erase command exists, in this order: its bucket objects, from the + keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no + memory content is left outside erasure; then its rows in every table with an `org` column + of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables + included; then its `loom.tenancy` row, on a session with `loom.tenancy_checked` set. +- **A name no request can carry** (not UTF-8, a NUL) found in a data table is skipped and + named; its rows stay outside placement. +- **An org found only in a log table** (`agent_events` and the tables derived from it, the + code search trace) is never placed: a log row keeps the org an unauthenticated caller + typed. Once that org is placed in another cell, its log rows here are outside its + erasure there. + +`loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, +smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left +unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available +yet. + +## After attach + +`loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres +role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an +org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or +a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a +writer added after `serving` passes attach's checks, which ran once; doctor is how it is +found. + +## Related + +- [Cells](../architecture/cells.md) +- [Troubleshooting: cells](troubleshooting.md#cells) +- [Upgrades](upgrades.md) diff --git a/docs/operations/postgres.md b/docs/operations/postgres.md index d087a158c4..895086c474 100644 --- a/docs/operations/postgres.md +++ b/docs/operations/postgres.md @@ -44,6 +44,10 @@ pooler refuses or drops (its org triggers then refuse Loom's own writes for an o cell holds no row for, and the `loom.tenancy` guard its tenancy row for each newly placed org), and `loom doctor` names a session that lost it. Scripts and bulk loaders never set `session_replication_role = replica` on a cell: it skips the org triggers and the guard. +No role is granted `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` +on `loom.tenancy_checked`, or membership in `pg_write_server_files` or +`pg_execute_server_program` on a cell: `loom cell attach` refuses such a role, and +`loom doctor` warns on one ([attaching the first cell](cell-attach.md)). Every replica of the deployment, and its worker process, sets the same values. The full list, with the admission and delivery settings, is in the diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 8caf8c789d..50dd5fc46a 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -320,6 +320,14 @@ checkout. answers them from one read per replica per `LOOM_ROUTER_DIRECTORY_CACHE_S` ([cells](../architecture/cells.md#the-heartbeats-keys)). +18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and + CI configuration of the cell (step 16), rotate the ClickHouse credential they write + with: a new user or password for the cell's servers and CLIs, the old one dropped. No + CLI sends a heartbeat, so attach cannot see a job still configured with `LOOM_CELL` + unset; the rotation cuts off any the rollout missed before it can write an org's rows + past attach's last scan. Name the new user with `--ch-allow-user` + ([attaching the first cell](./cell-attach.md)). + While a migration applies, the old build keeps serving: every migration is additive. The rules that let the two builds serve side by side are in [rolling upgrades](../architecture/rolling-upgrades.md). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..12bac9a65b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -315,7 +315,7 @@ python scripts/codegen/docs_reference.py | `loom bot create` | Register a bot: its namespaces, and a key (--generate-key or --public-key) and/or a CI trust (--issuer, --audience, --claim). | `--name`, `--org`, `--namespace-role`, `--generate-key`, `--public-key`, `--issuer`, `--audience`, `--claim`, `--jwks-uri`, `--max-token-age`, `--max-access-token-ttl`, `--max-agents`, `--agent-claim`, `--platform-admin`, `--replace` | | `loom bot list` | Every bot, active or disabled. | | | `loom bot disable` | Refuse its proofs and revoke the tokens it holds. | `--name` | -| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--token` | +| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--pg-allow-role`, `--stray-orgs`, `--stray-orgs-budget`, `--token` | | `loom report` | Print the bug form's Environment field: versions and the `loom doctor` checks. | `--open` | | `loom drops` | Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. | `--namespace`, `--org`, `--hours` | | `loom namespaces-gc` | List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per… | `--org`, `--prefix`, `--namespace`, `--exclude-prefix`, `--idle-hours`, `--include-registered`, `--erase`, `--no-purge`, `--skip-external`, `--batch`, `--show`, `--requester`, `--worker-pass` | @@ -330,6 +330,7 @@ python scripts/codegen/docs_reference.py | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | +| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/llms.txt b/llms.txt index ca465459d3..43b8421b0e 100644 --- a/llms.txt +++ b/llms.txt @@ -104,6 +104,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. +- [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 368f74ebf8..c8b72fb687 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -1,6 +1,6 @@ """`loom cell attach`: turn a prepared first cell into a `serving` one (docs/architecture/cells.md, "Attaching a cell"; the runbook is -docs/operations/cells.md). +docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: @@ -121,7 +121,7 @@ def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: Refusal( f"org of {len(org.encode())} bytes ({org[:64]}…)", f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " - "remove it first (docs/operations/cells.md, the over-long org procedure)", + "remove it first (docs/operations/cell-attach.md, the over-long org procedure)", ) for org, sources in sorted(found.too_long.items()) ] @@ -174,7 +174,8 @@ def __init__( self.row: dict[str, Any] = {} self.mode = "" self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) - self.keys_fp = "" + self.keys_fp = "" # the live processes' one root-secret fingerprint, last read + self.begun_keys_fp = "" # the one this run's `attach/begin` wrote self.report: dict[str, Any] = {"cell": self.name} # -- the lock session -------------------------------------------------------------- @@ -410,6 +411,7 @@ async def begin(self) -> None: try: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.router.call("/attach/begin", body)} + self.begun_keys_fp = self.keys_fp finally: await _delete_nonces(self.ch, nonces) @@ -523,7 +525,7 @@ async def _read() -> list[beats.Beat]: verdict = await beats.settle( _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() ) - if verdict.keys_fps != {self.keys_fp}: + if verdict.keys_fps != {self.begun_keys_fp}: raise _refused( "processes", "the root-secret fingerprint changed during attach (a root-secret rotation): " diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index cf83fa8f27..4a820f652b 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -286,7 +286,7 @@ async def _bucket() -> None: try: await step() except Exception as e: # noqa: BLE001 — every nonce is tried; a stale one holds no data - print(f"loom cell prepare: the probe nonce in {where} stays: {e}", file=sys.stderr) + print(f"loom cell: the probe nonce in {where} stays: {e}", file=sys.stderr) def _register_timeout() -> httpx.Timeout: diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..3e88dbafca 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -397,8 +397,8 @@ "options: -h, --help show this help message and exit --name NAME" ], "loom doctor": [ - "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--token TOKEN]", - "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --token TOKEN Override the client's token." + "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--pg-allow-role PG_ALLOW_ROLE] [--stray-orgs] [--stray-orgs-budget STRAY_ORGS_BUDGET] [--token TOKEN]", + "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --pg-allow-role PG_ALLOW_ROLE On a cell: a Postgres role that may make an org held (the servers' role). --stray-orgs On a cell: also read the orgs its stores hold without a tenancy row. --stray-orgs-budget STRAY_ORGS_BUDGET Seconds --stray-orgs may read for, smallest tables first (default 60). --token TOKEN Override the client's token." ], "loom report": [ "usage: loom report [-h] [--open]", @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare} ...", - "positional arguments: {credential,limit,which,remove,prepare} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", + "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -459,6 +459,10 @@ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell attach": [ + "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..3d597c1e03 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -11213,6 +11213,9 @@ "json", "client", "server_url", + "pg_allow_role", + "stray_orgs", + "stray_orgs_budget", "token" ] } @@ -11280,6 +11283,51 @@ "help": "Override the client's server URL.", "metavar": null }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "On a cell: a Postgres role that may make an org held (the servers' role).", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs" + ], + "dest": "stray_orgs", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "On a cell: also read the orgs its stores hold without a tenancy row.", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs-budget" + ], + "dest": "stray_orgs_budget", + "action": "_StoreAction", + "default": 60.0, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds --stray-orgs may read for, smallest tables first (default 60).", + "metavar": null + }, { "option_strings": [ "--token" @@ -12903,6 +12951,211 @@ "metavar": null } ] + }, + "attach": { + "prog": "loom cell attach", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_attach:cmd_attach" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "wait", + "scan_budget", + "dry_run", + "ch_allow_user", + "ch_allow_user_directory", + "ch_allow_object", + "ch_allow_replica", + "pg_allow_role", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds the waits share", + "metavar": null + }, + { + "option_strings": [ + "--scan-budget" + ], + "dest": "scan_budget", + "action": "_StoreAction", + "default": 7200, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds one full org scan may take", + "metavar": null + }, + { + "option_strings": [ + "--dry-run" + ], + "dest": "dry_run", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run)", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user" + ], + "dest": "ch_allow_user", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user that may write the configured databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user-directory" + ], + "dest": "ch_allow_user_directory", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user directory the check cannot read", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-object" + ], + "dest": "ch_allow_object", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse object (db.name) that writes the databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-replica" + ], + "dest": "ch_allow_replica", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A replica outside the cluster (db.table:replica)", + "metavar": null + }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A Postgres role that may make an org held (the servers' role)", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] } }, "aliases_of": { @@ -12910,7 +13163,8 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", - "prepare": "loom cell prepare" + "prepare": "loom cell prepare", + "attach": "loom cell attach" }, "choices_help": [ { @@ -12937,6 +13191,11 @@ "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", "metavar": "prepare" + }, + { + "dest": "attach", + "help": "Attach the prepared first cell: place its orgs, serve.", + "metavar": "attach" } ] } diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 015cebd948..1c725205e6 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -680,3 +680,20 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: def test_refusal_text() -> None: assert str(Refusal("a", "b")) == "a: b" + + +@pytest.mark.asyncio +async def test_a_root_secret_change_during_attach_refuses_at_the_end( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.begun_keys_fp = "before" + + async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: + return beats.Verdict(live=1, keys_fps={"after"}) + + monkeypatch.setattr(beats, "settle", settle) + with pytest.raises(Refused, match="a root-secret rotation"): + await run.final_checks(beats.Watch(None, "", "first")) + run.begun_keys_fp = "after" + await run.final_checks(beats.Watch(None, "", "first")) From 738292142572fbd9c971f3274e62ccdc34075748 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:35:26 +0000 Subject: [PATCH 03/25] The widened heartbeat window lives with attach, so server.core imports no boot or insights module instances.read_recent takes the window it reads back; attach's heartbeat module computes it from the live window and a stopping process's drain and retirement budgets. PRUNE_AFTER_S is marked as the contract it is. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach_beats.py | 14 ++++++++++++-- server/core/instances.py | 25 ++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a2fc37701d..d1d938a46b 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -32,6 +32,16 @@ StoppedUnset = Literal["refuse", "wait"] +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return instances.live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + @dataclass(frozen=True) class Beat: """One process: its instance row (`live`: inside the live window and not stopped) and @@ -48,7 +58,7 @@ class Beat: async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: - rows = await instances.read_recent(ch, since=since) + rows = await instances.read_recent(ch, window_s=widened_window_s(), since=since) ids = [str(r["instance_id"]) for r in rows] cells = await ch.query_all( f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " @@ -236,7 +246,7 @@ async def settle( def watch_period_s() -> float: """`T'`: how long a row whose window reaches past the rescan's start is kept before the leader's sweep may drop it, less a margin for clock skew.""" - return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S class Watch: diff --git a/server/core/instances.py b/server/core/instances.py index bb47bfd945..80e3e76896 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -56,7 +56,7 @@ # How old a row's last beat is before the leader drops its key. A contract across builds, # not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader # that sweeps them may run another build, so a change is a design change of its own. -PRUNE_AFTER_S = 3600 +PRUNE_AFTER_S = 3600 # not a setting: a contract across builds # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -134,16 +134,6 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) -def widened_window_s() -> float: - """The live window plus what a stopping process still does after its stop beat: it - drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, - stopped or not, is younger than this may still write.""" - from server.boot import SHUTDOWN_DRAIN_BUDGET_S - from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S - - return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S - - def boot_allowance_s() -> float: """How long a booting process takes to report its first cell row and its first router answer: two beats and one router read.""" @@ -217,10 +207,12 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out -async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: - """Every process of every deployment sharing this ClickHouse whose widened window - (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or - past now without one: stopped rows included, which `read_live` drops. `stopped` is its +async def read_recent( + ch: Any, *, window_s: float, since: str | None = None +) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose last beat plus + `window_s` reaches past `since` (a ClickHouse `DateTime64(3)` as text), or past now + without one: stopped rows included, which `read_live` drops. `stopped` is its newest row's flag and `live` whether that row is inside the live window and not stopped, both judged in ClickHouse.""" rows = await ch.query_all( @@ -238,7 +230,7 @@ async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, An """, params={ "live": int(live_window_s() * 1000), - "wide": int(widened_window_s() * 1000), + "wide": int(window_s * 1000), "since": since or "", }, ) @@ -397,5 +389,4 @@ def _reset_for_tests() -> None: "start", "stop", "tick", - "widened_window_s", ] From 257597973f82ed821f500576b9fd14700eeee9f8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:51:59 +0000 Subject: [PATCH 04/25] ensure_table's integration test lands with attach, its caller Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- .../integration/test_cell_org_triggers_pg.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index e49dde4426..a47f4baec5 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -714,6 +714,32 @@ async def test_missing_names_a_guard_disabled_narrowed_or_calling_another_functi assert "loom.tenancy_refuse_unmarked() has a body or settings no migration writes" in texts +@pytest.mark.asyncio +async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_asked() -> None: + """Attach creates a first cell's triggers in `recording`, one table at a time; a boot's + `ensure` never does, and the function passes every row there.""" + async with _cell("recording") as (dsn, conn): + with pytest.raises(RuntimeError, match="recording"): + await cell_triggers.ensure_table(conn, "scope_generations") + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + assert await _cell_triggers(conn) == { + ("scope_generations", "tenancy_insert", "O"), + ("scope_generations", "tenancy_update", "O"), + ("tenancy", "tenancy_record", "O"), + } + # Again: nothing to do. + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + with pytest.raises(ValueError, match="no cell triggers"): + await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) + plain = await _unmarked(dsn) + try: + await plain.execute(_INSERT, "unheld-org", "s") # recording passes every row + await plain.execute(_HOLD, "new-org") # a first cell still recording records + finally: + await plain.close() + + @pytest.mark.asyncio @pytest.mark.parametrize("loom_cell", ["eu-1", ""]) async def test_the_cell_refresh_reads_the_marker_its_pool_sends( From 79c97d22d48abd072952f595a90067484cd89d72 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 07:48:23 +0000 Subject: [PATCH 05/25] Fix a double space in the failure table's pooler row Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 96580ec7d2..46e1439538 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1292,7 +1292,7 @@ the directory row. | A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | | A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | | A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | | A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | From f2fad614ff5bab7f3dbbf65ab7f416e88c9dcd91 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:10:11 +0000 Subject: [PATCH 06/25] Attach refuses a table its role cannot alter up front, charges --wait in wall time, and says when the mode commit's outcome is unknown The refusals name every table the trigger pass would alter whose owner role attach's role is not a member of, before any write; ensure_table and _run_ddl answer applied, busy or denied, so a denial during the pass refuses at once rather than after --wait as a busy lock. --wait is wall time: each bounded wait charges its elapsed time, lock tries included, and every lock try waits at most what is left; scan time stays outside it. From the mode commit on, a stop that comes before the durable read says the commit's outcome is unknown and to rerun attach, for a refusal and for any other error alike. The prune-window check runs with the refusals, before any write; an enforced resume runs one trigger pass; the lock session opens inside the block that closes it; the durable read retries a timeout or an outage only; refusals name no command that does not exist. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 78 +++--- server/cli/cell_attach_beats.py | 37 ++- server/cli/cell_attach_pg.py | 235 +++++++++++------- server/core/cell_triggers.py | 41 +-- tests/integration/test_cell_attach.py | 35 +++ .../integration/test_cell_org_triggers_pg.py | 7 +- tests/unit/test_cell_attach.py | 124 ++++++++- 7 files changed, 396 insertions(+), 161 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index c8b72fb687..ea9cf77d35 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -365,9 +365,9 @@ async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: out += await self.catalog_refusals() out += await self.durability_refusals() rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) - tables_pg = [f"loom.{r['relname']}" for r in rows] - out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await apg.own_role_refusals(self.lock_conn, list(rows)) out += await self.ch_limit_refusals() + out += beats.prune_window_refusals() if out: raise Refused(out) state, _ = await apg.read_mode(self.lock_conn) @@ -449,6 +449,8 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg + # From here on the mode may have risen: only the durable read can say. + self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: @@ -465,22 +467,29 @@ async def commit_mode(self) -> None: async def durable(self) -> int: """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's xmax, above the read's own xid.""" - from server.core import pg_profile + from server.core import pg, pg_profile - while True: - try: + with self.wait.counting(): + while True: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous before the durable read") - read = await apg.durable_read() + try: + read = await apg.durable_read() + except Exception as e: + # A try cut off by its bound, or an outage: its outcome is unknown, and + # the next try reads again. Anything else is no reason to retry. + outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) + if not (isinstance(e, TimeoutError) or outage): + raise + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused( + "postgres durable read", "did not commit within --wait" + ) from e + continue if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break - except Refused: - raise - except Exception as e: - self.log(f"the durable read failed ({type(e).__name__}); retrying") - if not await self.wait.pause(): - raise _refused("postgres durable read", "did not commit within --wait") from e if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -540,8 +549,6 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: await self.cheap_checks("wait") since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) watch = beats.Watch(self.ch, since, self.name) - if refusals := watch.refusals_up_front(): - raise Refused(refusals) task = asyncio.create_task(watch.run()) try: listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) @@ -567,7 +574,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): raise _refused( "postgres loom.cell_self", - "differs from the directory's fingerprints (a loom cell update between them?)", + "differs from the directory's fingerprints (a fingerprinted setting changed " + "during attach?)", ) state, _ = await apg.read_mode(self.lock_conn) if state != "enforced": @@ -596,7 +604,9 @@ async def done(self) -> bool: if state != "enforced": raise _refused( "postgres cell state", - f"the directory says serving and the cell reads {state}: run loom cell restore", + f"the directory says serving and the cell reads {state}: the directory and the " + "cell's Postgres disagree, as after a restore of one of them; attach lowers " + "and repairs neither", ) return True @@ -606,8 +616,8 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 - await self.open_lock() try: + await self.open_lock() await self.read_own_row() topology, names, listed = await self.refusals() if self.args.dry_run: @@ -626,7 +636,6 @@ async def run(self) -> int: await self.precommit(found) await self.commit_mode() else: - await self.triggers() await self.router.call( "/attach/enforced", {"deployment_id": self.row["deployment_id"]} ) @@ -695,18 +704,30 @@ def log(text: str) -> None: except Refused as e: for r in e.refusals: log(f"refusing: {r}") - mode = ( - "enforced, attaching: rerun attach to resume" - if run.mode == "enforced" - else ("unchanged" if not run.mode else f"{run.mode}") - ) - log(f"the cell stays {mode}") + log(mode_line(run.mode)) + return 1 + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + log(f"stopped ({type(e).__name__}: {e}); run it again") + log(mode_line(run.mode)) return 1 run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code +def mode_line(mode: str) -> str: + """What a stopped run left the cell's mode at, by what it last knew.""" + if mode == "enforced": + return "the cell stays enforced and attaching: rerun attach to resume" + if mode == "unknown": + return ( + "the mode commit's outcome is unknown: rerun attach, which resumes by the mode it reads" + ) + if mode == "recording": + return "the cell stays recording" + return "the cell's state is unchanged" + + def cmd_attach(args: argparse.Namespace) -> int: if problem := _args_problem(args): print(f"loom cell attach: {problem}", file=sys.stderr) @@ -719,14 +740,7 @@ def cmd_attach(args: argparse.Namespace) -> int: from server.core.ch import statement_timeout async def _run(ch: Any) -> int: - try: - return await attach(ch, args, url) - except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes - print( - f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", - file=sys.stderr, - ) - return 1 + return await attach(ch, args, url) # Every ClickHouse statement, the connect included, may run as long as a scan may. with statement_timeout(args.scan_budget): diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index d1d938a46b..a9c6e80377 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -209,6 +209,16 @@ async def settle( """Read and judge until nothing is waited on: Refused at the first refusal, and once `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended once, to `extend_s` past the first sight of the last process first seen booting.""" + with wait.counting(): + return await _settle(read, judge_beats, wait, extend_s) + + +async def _settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + extend_s: float, +) -> Verdict: first_seen: dict[str, float] = {} extended = False while True: @@ -220,7 +230,7 @@ async def settle( raise Refused(refusals) return verdict for i in verdict.booting: - first_seen.setdefault(i, wait.spent) + first_seen.setdefault(i, wait.used()) if await wait.pause(): continue if extend_s and not extended and verdict.booting: @@ -249,6 +259,20 @@ def watch_period_s() -> float: return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S +def prune_window_refusals() -> list[Refusal]: + """A heartbeat interval so long that the leader may sweep a stopped process's rows + before step 4's watch reads them refuses before any write.""" + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + class Watch: """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at @@ -265,17 +289,6 @@ def __init__( self.gap = False self._good = clock() - def refusals_up_front(self) -> list[Refusal]: - if watch_period_s() < 2 * _READ_TIMEOUT_S: - return [ - Refusal( - "LOOM_INSTANCE_HEARTBEAT_SECONDS", - "so long that a stopped process's rows can be swept before attach reads " - "them: lower it", - ) - ] - return [] - async def read_once(self) -> list[Beat]: try: async with asyncio.timeout(_READ_TIMEOUT_S): diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 6ee4432907..076dc4a196 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -10,8 +10,9 @@ from __future__ import annotations import asyncio +import contextlib import time -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -23,38 +24,59 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval -# The migration lock's wait a trigger try (a boot holds it for its migrations). -_MIGRATION_LOCK_TIMEOUT = "10s" -# The mode commit's lock wait a try, and the statement bound above it. -_MODE_LOCK_TIMEOUT = "2s" -_MODE_STATEMENT_TIMEOUT = "5s" +# The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a +# try never waits longer than --wait has left. +_MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# The mode commit's lock wait a try, in ms, and its statement bound's margin above it. +_MODE_LOCK_MS = 2_000 # not a setting: one try's bound +_MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} class Wait: - """`--wait`: the seconds attach's waits share, counting only the time spent waiting - (never scan time).""" + """`--wait`: the wall-clock seconds attach's bounded waits share. Each wait runs inside + `counting()`, which charges its elapsed time, lock tries included; time outside every + wait (a scan) is never charged.""" def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: self.seconds = float(seconds) self.spent = 0.0 self._clock = clock + self._since: float | None = None + + def used(self) -> float: + """The seconds charged so far, the wait running now included.""" + running = 0.0 if self._since is None else self._clock() - self._since + return self.spent + running def left(self) -> float: - return max(0.0, self.seconds - self.spent) + return max(0.0, self.seconds - self.used()) + + @contextlib.contextmanager + def counting(self) -> Iterator[None]: + """Charge the block's wall time (a block inside another is charged once).""" + if self._since is not None: + yield + return + self._since = self._clock() + try: + yield + finally: + self.spent += self._clock() - self._since + self._since = None - def now(self) -> float: - return self._clock() + def bound_ms(self, most_ms: int) -> int: + """A lock timeout for one try: `most_ms`, or what is left when that is less.""" + return min(most_ms, int(self.left() * 1000)) async def pause(self, seconds: float = POLL_S) -> bool: - """Wait `seconds` (at most what is left) and count them; False once none is left.""" + """Wait `seconds` (at most what is left); False once none is left.""" step = min(seconds, self.left()) if step <= 0: return False - start = self._clock() - await _sleep(step) - self.spent += max(step, self._clock() - start) + with self.counting(): + await _sleep(step) return True @@ -137,7 +159,7 @@ async def take_lock(conn: Any, what: str) -> None: [ Refusal( "attach lock", - f"held by {holder}: an attach, prepare or update is running; {what} " + f"held by {holder}: an attach or a prepare is running; {what} " "runs one at a time", ) ] @@ -364,14 +386,22 @@ async def session_passes() -> bool: """ -async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: - """What attach's own role lacks (the guarded tables' owner roles are `missing`'s - `owner` defects), and a server that allows prepared transactions.""" +async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: + """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` + read: its grants, `SELECT` on every org table, and membership in the owner role of every + table it gives triggers (or the trigger pass could not alter it); and a server that + allows prepared transactions.""" role = await conn.fetchval("SELECT current_user") lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] - for table in tables: - if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): - lacks.append(f"SELECT on {table}") + for r in tables: + if not await conn.fetchval( + "SELECT has_table_privilege($1, 'SELECT')", f"loom.{r['relname']}" + ): + lacks.append(f"SELECT on loom.{r['relname']}") + planned = set(cell_triggers.planned_names(tables)) + for r in tables: + if r["relname"] in planned and not r["owned"]: + lacks.append(f"membership in the owner role of loom.{r['relname']}") out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] out += await prepared_refusals(conn) return out @@ -400,51 +430,66 @@ async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> """Give every planned table its triggers (`cell_triggers.ensure_table` with `recording_first`), one try at a time under the migration lock, retrying a table whose lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield - by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" - from server.core.schema.pg_migrate import _LOCK_KEY - + by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. + Then `missing`: any defect refuses.""" deadlock_ms = int( await conn.fetchval( "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " "FROM pg_settings WHERE name = 'deadlock_timeout'" ) ) - out: list[Refusal] = [] - for table in await cell_triggers.planned_tables(conn): - yielded = False - while True: - lock_timeout = cell_triggers.LOCK_TIMEOUT - holders = await cell_triggers.holders(conn, table) - if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_timeout, yielded = f"{deadlock_ms + 500}ms", True - done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) - if done: - break - if not await wait.pause(): - out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) - break - log(f"loom.{table}: its lock is busy; retrying") - if out: - return out + with wait.counting(): + for table in await cell_triggers.planned_tables(conn): + if refusal := await _ensure_table(conn, table, deadlock_ms, wait, log): + return [refusal] return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] -async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: - await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") +async def _ensure_table( + conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] +) -> Refusal | None: + yielded = False + while True: + lock_ms = 100 + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_ms, yielded = deadlock_ms + 500, True + lock_ms = wait.bound_ms(lock_ms) + result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) + if result == "applied": + return None + if result == "denied": + return Refusal( + f"postgres loom.{table}", + "attach's role cannot alter it: make the role a member of its owner role", + ) + if not await wait.pause(): + return _busy_refusal(table, await cell_triggers.holders(conn, table)) + log(f"loom.{table}: its lock is busy; retrying") + + +async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: + """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) + if migration_ms <= 0: + return "busy" + await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: - await conn.execute(_MIGRATION_LOCK, key) + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return False # a boot migrating: the next try + return "busy" # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) finally: - await conn.execute(_MIGRATION_UNLOCK, key) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: @@ -487,49 +532,56 @@ async def commit_enforced( log: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock - session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, - within `wait`; `confirm_lock` before each try and each termination (it refuses when - another run holds the attach lock), and `ready` before each try (False skips the try - while a process boots). After a failed try the in-flight mode transaction, found by its - lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the - durable read's to say.""" - while True: - await confirm_lock() - if await ready(): - try: - async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") - await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", f"the mode update read {status}" - ) - ] - ) - return - except Refused: - raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less + when `wait` has less left), within `wait`; `confirm_lock` before each try and each + termination (it refuses when another run holds the attach lock), and `ready` before + each try (False skips the try while a process boots). After a failed try the in-flight + mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; + whether the commit landed is the durable read's to say.""" + with wait.counting(): + while True: + await confirm_lock() + lock_ms = wait.bound_ms(_MODE_LOCK_MS) + if lock_ms > 0 and await ready(): + if await _mode_try(conn, lock_ms): + return log("the mode's lock is busy; retrying") - await end_mode_transactions(conn, confirm_lock) - if not await wait.pause(): - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", - "its lock was not taken within --wait: a long transaction reads it " - "FOR SHARE; rerun attach", - ) - ] + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads " + "it FOR SHARE; rerun attach", + ) + ] + ) + + +async def _mode_try(conn: Any, lock_ms: int) -> bool: + """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + return True + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: @@ -626,6 +678,11 @@ async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: """Wait until no backend of the cell's database holds an xid or a snapshot older than `xmax`; Refused at once on a prepared transaction or an unreadable row, and past `wait` naming each backend waited on and each other database's horizon holder.""" + with wait.counting(): + await _wait_transactions(conn, xmax, wait) + + +async def _wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: while True: if refusals := await prepared_refusals(conn): raise Refused(refusals) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 9c2c9755ae..516dee4cc1 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any +from typing import Any, Literal from server.core import cell, pg, readiness @@ -92,6 +92,9 @@ AND l.mode NOT IN ('AccessShareLock', 'RowShareLock') """ +# What a table's DDL came to (`ensure_table`, `_run_ddl`). +Applied = Literal["applied", "busy", "denied"] + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -242,7 +245,12 @@ async def ensure(conn: Any) -> None: async def planned_tables(conn: Any) -> list[str]: """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" - return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + return planned_names(await conn.fetch(ORG_TABLES)) + + +def planned_names(rows: list[Any]) -> list[str]: + """`planned_tables` from rows `ORG_TABLES` read.""" + return [r["relname"] for r, _ in _planned(rows)] async def ensure_table( @@ -252,14 +260,14 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: +) -> Applied: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function - passes every row and a boot never creates them. False when the DDL was not applied (its - lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); - True when it was, or nothing was needed. Raises for a table outside the planned set, or - in another state.""" + passes every row and a boot never creates them. `applied` when the DDL ran or nothing + was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, + `denied` when this role cannot alter the table. Raises for a table outside the planned + set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -270,7 +278,9 @@ async def ensure_table( raise ValueError(f"loom.{table} carries no cell triggers") ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) - return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + if not sqls: + return "applied" + return await _run_ddl(conn, table, sqls, lock_timeout, attempts) def _table_statements( @@ -293,10 +303,11 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The - boot serves past the two causes cells.md names (its lock not taken, a table this role - cannot alter); any other error fails it, as a failed read of the state does.""" +) -> Applied: + """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the + two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: + a table this role cannot alter), which a boot serves past; any other error fails it, as + a failed read of the state does.""" import asyncpg for attempt in range(attempts): @@ -305,12 +316,12 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return True + return "applied" except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return False + return "denied" except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -323,7 +334,7 @@ async def _run_ddl( "; ".join(sqls), await _holders(conn, table), ) - return False + return "busy" async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 406d277646..9bfe3fe9bf 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -633,3 +633,38 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( + cell_stores, +) -> None: + from server.core import cell_triggers + + await _prepare(cell_stores, _Network()) + role = f"it_o_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA loom TO {role}" + ) + await conn.execute(f"SET ROLE {role}") + rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) + refusals = await cell_attach_pg.own_role_refusals(conn, rows) + lacking = [r.reason for r in refusals if "owner role" in r.reason] + assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking + assert any("loom.scope_generations" in r for r in lacking) + # The trigger pass tells the cause apart: denied, not a busy lock. + assert ( + await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + == "denied" + ) + await conn.execute("RESET ROLE") + assert await _mode() == "recording" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index a47f4baec5..b2fb66a467 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -721,15 +721,16 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a async with _cell("recording") as (dsn, conn): with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + ensure = cell_triggers.ensure_table + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert await ensure(conn, "tenancy", recording_first=True) == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 1c725205e6..b57a40eb57 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -222,12 +222,72 @@ async def forever() -> list[beats.Beat]: @pytest.mark.asyncio -async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: - wait = apg.Wait(2, clock) - clock.now += 100 # a scan's time - assert await wait.pause() and await wait.pause() - assert not await wait.pause() - assert wait.spent == 2.0 +async def test_wait_is_wall_time_lock_tries_included_and_no_scan(clock: _Clock) -> None: + """Each mode-commit try waits its lock timeout before it times out: the tries and the + pauses between them end within --wait, and time outside the wait is never charged.""" + asked: list[str] = [] + + def lock(sql: str, *_a: Any) -> str: + if "lock_timeout" in sql: + asked.append(sql) + clock.now += int(sql.split("'")[1].removesuffix("ms")) / 1000 + raise _Timeout() + return "OK" + + conn = FakeConn( + [ + ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + + async def nothing() -> None: + pass + + async def ready() -> bool: + return True + + wait = apg.Wait(4, clock) + clock.now += 100 # a scan before the wait + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + assert clock.now - 100 <= 4 and wait.spent <= 4 + # A 2 s try, a 1 s pause, then a try that waits only the second left. + assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] + + +@pytest.mark.asyncio +async def test_the_trigger_pass_refuses_at_once_a_table_it_cannot_alter( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def ensure_table(*_a: Any, **_kw: Any) -> str: + return "denied" + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", holders) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn([("deadlock_timeout", 1000)]) + wait = apg.Wait(60, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert refusal.subject == "postgres loom.scope_generations" + assert "cannot alter it" in refusal.reason and wait.spent < 1 + + async def busy(*_a: Any, **_kw: Any) -> str: + clock.now += 0.1 + return "busy" + + monkeypatch.setattr(cell_triggers, "ensure_table", busy) + wait = apg.Wait(3, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert "not taken within --wait" in refusal.reason and wait.spent <= 3 # -- the rescan's watch ---------------------------------------------------------------- @@ -272,9 +332,9 @@ def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) - assert beats.Watch(None, "", "first").refusals_up_front() + assert beats.prune_window_refusals() monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) - assert beats.Watch(None, "", "first").refusals_up_front() == [] + assert beats.prune_window_refusals() == [] # -- batches, ordering, arguments -------------------------------------------------------- @@ -491,10 +551,19 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ("max_prepared_transactions", 5), ] ) - refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + tables = [ + {"relname": "memory_current", "relkind": "r", "owned": True}, + # Its grants are held, but not its owner role's membership: the trigger pass could + # not alter it, so it refuses before any write. + {"relname": "scope_generations", "relkind": "r", "owned": False}, + {"relname": "tenancy", "relkind": "r", "owned": True}, + ] + refusals = await apg.own_role_refusals(conn, tables) assert [str(r) for r in refusals] == [ "postgres role attacher: attach's role lacks membership in pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres role attacher: attach's role lacks membership in the owner role of " + "loom.scope_generations", "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " "session, so step 4 could not wait for it; set it to 0", ] @@ -674,7 +743,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] assert await run.done() state[0] = "recording" - with pytest.raises(Refused, match="run loom cell restore"): + with pytest.raises(Refused, match="attach lowers and repairs neither"): await run.done() @@ -697,3 +766,38 @@ async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: await run.final_checks(beats.Watch(None, "", "first")) run.begun_keys_fp = "after" await run.final_checks(beats.Watch(None, "", "first")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [Refused([Refusal("postgres", "cut")]), OSError("gone")]) +async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outcome_is_unknown( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: Exception +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def landed(*_a: Any) -> None: + pass # the commit landed; its answer may not have + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def run(self: Any) -> int: + self.row = {"deployment_id": "dep-1"} + await self.commit_mode() + raise failure # what stops the run before the durable read finishes + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "commit_enforced", landed) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "run", run) + url = httpx.URL("http://127.0.0.1:7900") + assert await cell_attach.attach(None, _args(), url) == 1 + err = capsys.readouterr().err + assert "the mode commit's outcome is unknown: rerun attach" in err + assert "stays recording" not in err From dcf7996e290d83efdade60734379ae1c33bee5d4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:17:07 +0000 Subject: [PATCH 07/25] Doctor's stray-org read names a table it cannot read and goes on, and the attach docs describe one step numbering and today's procedure --stray-orgs labels only a spent budget as tables left unchecked; any other refusal names its table and the read continues. A failed catalog read reports both of attach's Postgres checks as not read. The runbook and doctor describe removing a stray or over-long org by hand, and name no command that does not exist yet. cells.md labels the first attach's list with the design's step names, so step 1 to step 4 mean the same everywhere, rewrites the stuck-process failure row and the planned tests for a mode that only rises, and says --wait is wall time. The trigger-cost harness uses cell_triggers.create_statement and deletes the rows its inserts wrote. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 21 +- docs/architecture/cells.md | 405 +++++++++++++------------- docs/operations/cell-attach.md | 19 +- server/cli/cell_attach.py | 41 +-- server/cli/doctor_cell.py | 36 ++- server/core/cell_triggers.py | 5 + tests/integration/test_cell_attach.py | 20 ++ tests/unit/test_cell_attach.py | 27 ++ 8 files changed, 325 insertions(+), 249 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index e90fbb32fa..4363a810d6 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1,9 +1,10 @@ """The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` -guard stays). Run it against an attached cell's server, which must not serve anything else, -with a writer token for a placed org; it drops and recreates the cell's org triggers, so -never against a deployment in use. Needs no provider. +guard stays), deleting the rows its inserts wrote at the end. Run it against an attached +cell's server, which must not serve anything else, with a writer token for a placed org; +it drops and recreates the cell's org triggers, so never against a deployment in use. +Needs no provider. PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ --base-url http://127.0.0.1:7932 --pg-dsn \\ @@ -36,7 +37,7 @@ async def _triggers(dsn: str, on: bool) -> None: for name in cell_triggers.ORG_TRIGGERS: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') if on: - await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + await conn.execute(cell_triggers.create_statement(table, name)) enabled = await conn.fetchval( "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", list(cell_triggers.ORG_TRIGGERS), @@ -111,6 +112,18 @@ async def main() -> None: ) finally: await _triggers(args.pg_dsn, True) + await _clean(args) + + +async def _clean(args: argparse.Namespace) -> None: + """Delete the rows the inserts wrote (the `memory.set` calls' rows stay the org's).""" + conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) + try: + await conn.execute( + "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + ) + finally: + await conn.close() if __name__ == "__main__": diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 46e1439538..3ea7ddab80 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -682,7 +682,8 @@ mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_ this database (a pid captured at connect is wrong behind a connection path that swaps backends), and waits until the lock clears. Whether the commit landed is decided by a durable read of the mode, whose commit waits for the standby, since a commit ended while -it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +it waited is on the primary alone ([attaching a cell](#attaching-a-cell): step 3 and the +durable read). The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in @@ -881,14 +882,13 @@ read-only scope, lists any tenancy row the directory does not place. yet, a later cell's configured orgs at attach and a changed one at `loom cell update` included, passes the gate, a closed cell and `placements_max` included, so a later cell closed or full before attach refuses the attach, naming the org. -- **The first cell records until `enforced` as a single deployment does.** In - `recording`, which lasts until step 3 writes `enforced` to the cell's mode (and again - whenever step 3 or step 4 returns the cell to it), a verified request for a new org - writes a tenancy row with no placement call, so the router's rate, `placements_max` - and `loom cell limit --closed` do not apply: new orgs arrive as they do on a single - deployment, bounded by its own signup, issuer and JIT settings, and step 2 or step 4 - places them unrefused. `loom doctor` lists the recorded orgs that hold no data, so the - operator sees what attach will place. +- **The first cell records until `enforced` as a single deployment does.** In `recording`, + which lasts until step 3 writes `enforced` to the cell's mode (the mode only rises after + it), a verified request for a new org writes a tenancy row with no placement call, so the + router's rate, `placements_max` and `loom cell limit --closed` do not apply: new orgs + arrive as they do on a single deployment, bounded by its own signup, issuer and JIT + settings, and step 2 or step 4 places them unrefused. `loom doctor` lists the recorded + orgs that hold no data, so the operator sees what attach will place. - **Configured orgs are placed at attach.** The cell's `oidc_jit_org`, the `self_signup_org` of a cell with signup on, and its code orgs (`LOOM_CODE_ORG` and the org of each `LOOM_CODE_REPOS_JSON` entry the code config keeps) and tracker org are @@ -986,8 +986,8 @@ The first attach (`loom cell attach`, its procedure on [attaching the first cell](../operations/cell-attach.md)) turns the existing deployment into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at `attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an -advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and -`loom cell update` take the same lock), after checking that the session is one direct +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare +takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would refuse every later run, and attach's tenancy rows would be refused as an unchecked writer's. A lock found lost between steps is taken again (refusing when another run holds @@ -1034,74 +1034,76 @@ It refuses, before any write, naming each: `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user reads. -Then, in order: - -1. `attach/begin`, on every run, before any long step: the router probes the stores - attach wrote its nonces to, checks the upstream, and writes the fingerprints and - `keys_fp`; a directory restored to before it is written again. -2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, - through `attach/enforced` again, to the durable read. -3. Create the triggers itself, on every run, before step 2's scan and again just before the - mode commit (so a table a migration added during the scan is guarded from the commit - on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock - and retried a second apart within `--wait`; an autovacuum holding it gets one try just - past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then - `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the - triggers, which pass every row there. -4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads - every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an - `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with - the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not - read: a log row keeps the org a caller typed before any check, so placing a log-only - org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such - orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row - is written; a name no request can carry is skipped and named. Then the tenancy rows of - the orgs found, on a marked session, and no placement: unscoped callers are answered - from the tenancy rows, and placing now would move the point of no return earlier. -5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session - checks and the standbys again, and the scan's elapsed time within half of - `--scan-budget`, since step 4 reads the same tables again. Each refuses in - `recording`, which can still be backed out. -6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on - `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock - (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database - alone) and ended. From the commit on the mode only rises: every later refusal leaves - the cell `enforced` and `attaching`, and a rerun resumes from the durable read. -7. The durable read, in every run that reads `enforced`: the mode and the `first` row read - in a transaction that emits a transactional logical message, whose commit waits for - the synchronous standbys, with the standbys checked attached just before and after. - Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED - writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a - snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is - assigned after the mode commit. -8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the - keys, unmarked, or with no router where it serves, refuses at once. No process depends - on its cached mode (a request records a new org through `tenancy._RECORD`, which reads - the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. -9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling - one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` - (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach - refuses, naming each, and each backend of another database holding the horizon. A - prepared transaction, or a row of the cell's database whose state is hidden, refuses - at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the - heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); - the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery - tables move rows to ClickHouse), and the queue and drain tables and the tables a view - reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, - in `attach/place` batches; the writer check again; and the final checks, under a floor - of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): - the heartbeats, read through the rescan every half of the time the leader keeps a - stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` - unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` - equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. - -`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 -hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would -be placed and skipped and the scan's time, and writes nothing; the router's checks at -`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is -done (exit 0), anything else refuses, naming `loom cell restore`. +Then, in order (the step numbers are the ones the rest of this page uses): + +- **`attach/begin`**, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +- **Step 1, the mode**: `recording`, or `enforced` on a resume, which goes from the + triggers, through `attach/enforced` again, to the durable read. +- **The triggers**: created by attach itself, on every run, before step 2 and again just + before step 3 (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +- **Step 2, the scan**, in `recording`: one statement per table, no long transaction (step 4 + reads every source again): the Postgres tables (`loom.tenancy` among them) by skip scans + on an `org`-led btree index, then the ClickHouse tables, each flushed and synced first, + with the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only org + would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such orgs. + An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row is + written; a name no request can carry is skipped and named. Then the tenancy rows of the + orgs found, on a marked session, and no placement: unscoped callers are answered from the + tenancy rows, and placing now would move the point of no return earlier. +- **The pre-commit checks**: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, + since step 4 reads the same tables again. Each refuses in `recording`, which can still be + backed out. +- **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE + lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock (only + attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) + and ended. From the commit on the mode only rises: every later refusal leaves the cell + `enforced` and `attaching`, and a rerun resumes from the durable read. +- **The durable read**, in every run that reads `enforced`: the mode and the `first` row + read in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. Then + `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED writer + that read `recording` holds an xid below `X`, and a REPEATABLE READ one a snapshot older + than `X`: the trigger functions are `VOLATILE`, and the message's xid is assigned after + the mode commit. +- **The heartbeats** again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends on + its cached mode (a request records a new org through `tenancy._RECORD`, which reads the + mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +- **Step 4**: wait, within `--wait`, while any backend of the cell's database but the + polling one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A prepared + transaction, or a row of the cell's database whose state is hidden, refuses at once. Then + the cheap checks (the writer check, `missing`, the Postgres checks, the heartbeats with + every stopped process with `LOOM_CELL` unset aged out, the standbys); the rescan of every + source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to + ClickHouse), and the queue and drain tables and the tables a view reads before the others + (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; + the writer check again; and the final checks, under a floor of their own (the larger of + what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the + rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), + refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to + the directory's fingerprints, one `keys_fp` equal to the one `begin` wrote, the mode + `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try +included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each +full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints +what would be placed and skipped and the scan's time, and writes nothing; the router's +checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: +`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's +Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1297,13 +1299,13 @@ the directory row. | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | | A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A process stuck during first attach | before step 3, attach refuses after `--wait` and the cell stays `recording`; after it, attach refuses, leaving the cell `enforced` and `attaching`, and a rerun resumes | | A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | | Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | | A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | | A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | | A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | -| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until they are removed by hand (the runbook's procedure); `loom doctor --stray-orgs` names it | | An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | | A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | | `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | @@ -1474,126 +1476,117 @@ An integration test with two cells and a router in CI's Postgres job: - every route, the batch entries of MCP and every CLI command included, refuses another cell's org and writes nothing in its Postgres or ClickHouse (the route, op-field and CLI censuses cover the rest); -- the first attach places every org with a row in any table, an org first seen during - the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a - bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` - of failed reads of its own rows or mode is gated and serves again once a read lands; a - pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer - request, passes its health probe, answers the probe route and runs no worker on an - attached cell and on a later cell between prepare and attach, keeps serving on a first - cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included; a CLI command - declaring an org with it unset is refused on an `enforced` or `partial` cell, an - org-free one (a token revocation) runs, also with Postgres down on a deployment never - prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds - with the cell's Postgres down; after `loom cell credential --cell --next` both - credentials place and read state, `--finish` refuses the old credential, for a cell - finished before its pods hold the new one the router refuses its calls with 401, its - request answers 503 writing nothing, and it recovers by recording the old hash again, - `--revert` drops a pending `next`, which is then refused, and the old credential keeps - placing, and a second `--next` while one is pending and a hash another cell records as - current or next are refused; `loom cell limit` and `loom cell credential --next`, - `--finish` and `--revert` refuse while the directory is unavailable; concurrent - placements at one under `placements_max` across two router replicas write one; a CLI - command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a - first cell once it is `enforced` (`attaching` included) and on a later cell, and on a - first cell in `recording` runs and records the org; attach with `placements_max` below - the orgs the first cell holds places them all, and a later cell's attach past its - quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step - 2 and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's - mode, a process whose cache still reads `recording` sends a new org's request to the - placement call, which a closed cell answers 503; a CLI command whose insert finds - `enforced` refuses, naming the org, and makes no placement call; after step 3's - refused wait or step 4's recheck returns the cell to `recording`, a new org is - recorded with no placement call, and one at a process whose cache still reads - `enforced` places through the gate until the cache expires; each ordering of attach's - write of the mode with a recording insert holds as stated above (running, starting - during the write, a stream of overlapping inserts each shorter than a try, with attach - setting `enforced` within a try or two, held across several tries, held until too - little of `--wait` is left, a stalled attach, a commit cut off by its timeout, a - commit whose connection is lost while it waits for the standby, read back only once - that backend has gone and written again durably if it reads `enforced`); a - `REPEATABLE READ` transaction whose first statement ran before `enforced` committed, - on attach's role or a second role, holds step 4 until it ends and its write of an - unheld org is then placed, one whose first statement ran after is refused by the - trigger, a transaction that starts after attach takes the lock but before its commit - is visible holds step 4, a transaction opened in `recording` that outlives one run's - `--wait` holds the rerun's step 4, after a refusal and after `loom cell restore` - raised the mode, an autovacuum worker on a loom table does not hold step 4 and a - background worker with a transaction open since `recording` does, a prepared - transaction (after a restart with `max_prepared_transactions` raised during the wait) - makes step 4 refuse at once, naming it, with the mode still `enforced` and a rerun - refused until it is resolved and the setting is back to 0, a write transaction open in - a second database on the same server holds step 4 and is named, a long read on the - synchronous standby, or on a second standby, with `hot_standby_feedback` on does not - hold it, nor does a base backup running across it, a write held open on a +- the first attach places every org with a row in any table, an org first seen during the + attach is placed in step 4, and so is one a direct Postgres insert wrote between step 2's + scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object + catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of + its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` + unset is unready, naming the setting, answers 503 to a bearer request, passes its health + probe, answers the probe route and runs no worker on an attached cell and on a later cell + between prepare and attach, keeps serving on a first cell in `recording`, and keeps the + last mode it read through a failed read, `recording` included; a CLI command declaring an + org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token + revocation) runs, also with Postgres down on a deployment never prepared, and a `loom +cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres + down; after `loom cell credential --cell --next` both credentials place and read + state, `--finish` refuses the old credential, for a cell finished before its pods hold the + new one the router refuses its calls with 401, its request answers 503 writing nothing, + and it recovers by recording the old hash again, `--revert` drops a pending `next`, which + is then refused, and the old credential keeps placing, and a second `--next` while one is + pending and a hash another cell records as current or next are refused; `loom cell limit` + and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is + unavailable; concurrent placements at one under `placements_max` across two router + replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set + refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a + later cell, and on a first cell in `recording` runs and records the org; attach with + `placements_max` below the orgs the first cell holds places them all, and a later cell's + attach past its quota or closed refuses, naming the org; a first cell with `--closed` and + `placements_max` set before step 2 attaches, and an org a request records between step 2 + and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a + process whose cache still reads `recording` sends a new org's request to the placement + call, which a closed cell answers 503; a CLI command whose insert finds `enforced` + refuses, naming the org, and makes no placement call; a refusal after step 3's write + leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; + each ordering of attach's write of the mode with a recording insert holds as stated above + (running, starting during the write, a stream of overlapping inserts each shorter than a + try, with attach setting `enforced` within a try or two, held across several tries, held + past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut + off by its timeout, and a commit whose connection is lost while it waits for the standby, + which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran + before `enforced` committed, on attach's role or a second role, holds step 4 until it ends + and its write of an unheld org is then placed, one whose first statement ran after is + refused by the trigger, a transaction that starts after attach takes the lock but before + its commit is visible holds step 4, a transaction opened in `recording` that outlives one + run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom + table does not hold step 4 and a background worker with a transaction open since + `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` + raised during the wait) makes step 4 refuse at once, naming it, with the mode still + `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write + transaction open in a second database on the same server holds step 4 and is named, a long + read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on + does not hold it, nor does a base backup running across it, a write held open on a `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a - long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database - past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the - first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` - above 0 refuses before step 1 with the mode still `recording`, while a later cell's - attach does not require either; a missing or disabled org trigger is refused by attach - and named by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode - row was deleted is refused and the state named by `loom doctor`; on that cell a pod - with `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an - org with it unset is refused, as a pod is; updating, deleting or truncating the - `first` row on an `enforced` cell is refused, also under - `session_replication_role = replica`, and a missing or disabled `first`-row trigger is - refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set - on a cell whose mode row is removed answers 503 to a bearer request and runs no worker - pass, and serves again after a rerun of `loom cell prepare` writes the row back while - it runs; a cell the directory registers with no state, tenancy or own row is named by - `loom doctor`, its processes with `LOOM_CELL` set stay unready, a process with it - unset serves as a single deployment, and `loom cell restore` writes both rows back, an - attached cell then serving again and an unattached one after a prepare rerun; a cell - with both rows absent beside a tenancy row, and one beside only its own row, has the - function refuse an unheld org (a boot creates no trigger there), every process gated - and `loom doctor` naming it, and a prepare rerun writes the rows back; a Postgres-only - restore of an attached first cell to before its prepare is named by `loom doctor` and - closed by `loom cell restore`; a prepare rerun during a pending `loom cell update` on - an attached cell, with processes on both values and `loom.cell_self` lost, writes both - values back and keeps both sets of processes serving, and one before attach refuses - the two configurations; a prepare rerun on an attached cell whose `current` is missing - and whose pods' configuration differs from the directory row refuses, and one before - attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a - credential hash that is neither the row's `current` nor its `next` is refused; a - prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode - row back; a cell with a mode row and no `first` row has the function refuse an unheld - org (a boot creates no trigger there), every process gated and `loom doctor` naming - it, and a prepare rerun writes the row back; a later cell whose mode row is set to - `recording` records nothing, its trigger refuses an unheld org, a process with - `LOOM_CELL` set there serves as in `enforced`, a pod with `LOOM_CELL` unset there is - unready and runs no worker, `loom cell prepare` names the row, a CLI command declaring - an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses - its state read; a build N-1 process's `memory.set` and delivery pass on a - never-prepared deployment after N's migration succeed and write no tenancy row, and on - the first cell in `recording` a boot creates no org trigger, and one created by hand - reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), - an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused - and one that hits an existing row of it is not, a marked session's every write passes, - and the triggers on an `enforced` cell cover every table erasure classifies with an - `org` column but the three exclusions; and an org-keyed write while attach holds the - mode table's lock returns at once, with new-org requests answering within their - deadline and mode-cache reads returning at once throughout; `loom cell restore` after - a refusal at step 3's lock leaves the cell `enforced`, and the resumed attach runs - from step 3's wait; a `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at - boot and named by `loom doctor`; `loom cell credential new` prints a credential of - that form and its SHA-256, with the directory and the stores unreachable; prepare and - `--next` refuse an argument that is not 64 lowercase hex characters, the credential - and its bare 128-character secret included, storing nothing; orgs a first cell records - in `recording` with no data are listed by `loom doctor`; a placement flood with no - quota set is held to the rate limit at each router replica, `loom doctor` names the - cell as one with no quota, and a call refused past it at one replica answers 503 with - a `Retry-After` of at most 60 s, after which its retry at that replica, with no other - call of the cell admitted between, is admitted; a cell's first prepare, with no - directory row yet, passes the deployment-id check; a pod or CLI that lost - `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the authority check; - the first cell's existing tokens, refresh tokens and OAuth clients keep working - through the cutover, a logout through the shared host revokes its refresh token; + transaction on one, opened in `recording` and writing nothing before `enforced`, a long + logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past + `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first + cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 + refuses before step 1 with the mode still `recording`, while a later cell's attach does + not require either; a missing or disabled org trigger is refused by attach and named by + `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted + is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset + is unready and runs no worker, and a CLI command declaring an org with it unset is + refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` + cell is refused, also under `session_replication_role = replica`, and a missing or + disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer + request and runs no worker pass, and serves again after a rerun of `loom cell prepare` + writes the row back while it runs; a cell the directory registers with no state, tenancy + or own row is named by `loom doctor`, its processes with `LOOM_CELL` set stay unready, a + process with it unset serves as a single deployment, and `loom cell restore` writes both + rows back, an attached cell then serving again and an unattached one after a prepare + rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, + has the function refuse an unheld org (a boot creates no trigger there), every process + gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a + Postgres-only restore of an attached first cell to before its prepare is named by `loom +doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell +update` on an attached cell, with processes on both values and `loom.cell_self` lost, + writes both values back and keeps both sets of processes serving, and one before attach + refuses the two configurations; a prepare rerun on an attached cell whose `current` is + missing and whose pods' configuration differs from the directory row refuses, and one + before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given + a credential hash that is neither the row's `current` nor its `next` is refused; a prepare + rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a + cell with a mode row and no `first` row has the function refuse an unheld org (a boot + creates no trigger there), every process gated and `loom doctor` naming it, and a prepare + rerun writes the row back; a later cell whose mode row is set to `recording` records + nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom +cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` + set refuses, naming it, and the router refuses its state read; a build N-1 process's + `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed + and write no tenancy row, and on the first cell in `recording` a boot creates no org + trigger, and one created by hand reads the state and looks nothing up, a write under + `session_replication_role = replica` for an unheld org is skipped (origin triggers), an + `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one + that hits an existing row of it is not, a marked session's every write passes, and the + triggers on an `enforced` cell cover every table erasure classifies with an `org` column + but the three exclusions; and an org-keyed write while attach holds the mode table's lock + returns at once, with new-org requests answering within their deadline and mode-cache + reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` + with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom +doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with + the directory and the stores unreachable; prepare and `--next` refuse an argument that is + not 64 lowercase hex characters, the credential and its bare 128-character secret + included, storing nothing; orgs a first cell records in `recording` with no data are + listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at + each router replica, `loom doctor` names the cell as one with no quota, and a call refused + past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its + retry at that replica, with no other call of the cell admitted between, is admitted; a + cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or + CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the + authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep + working through the cutover, a logout through the shared host revokes its refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the @@ -1607,12 +1600,12 @@ An integration test with two cells and a router in CI's Postgres job: application ends the person there, a platform-admin bot still reaches a cell whose SCIM deactivated everyone, and attach refuses a cell with unprovisioned live subjects or sharing another's OIDC client; -- an attach stopped in step 3 after it set `enforced`, before the wait passed, resumes - with the wait before the SCIM precondition, and one refused by the SCIM precondition - resumes from the wait too, the mode still `enforced`; a later attach refused before - its directory transaction leaves `attach_started_at` empty, so a rerun checks a new - cell's refusals again, and a resumed first attach finds its configured orgs placed in - this cell and carries on; +- an attach stopped after step 3's commit landed, before the durable read, says the commit's + outcome is unknown, and its rerun resumes by the mode it reads; one refused after the + durable read resumes from it, the mode still `enforced`; a later attach refused before its + directory transaction leaves `attach_started_at` empty, so a rerun checks a new cell's + refusals again, and a resumed first attach finds its configured orgs placed in this cell + and carries on; - a cell restored to before the first attach's `enforced` gets its mode back from `loom cell restore`; one whose `loom cell restore` was skipped answers 503 for an org it does not hold, another cell's org included, writes no tenancy row for it, and is diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 12a0cbe247..c51541d06d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -105,17 +105,16 @@ and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unse always goes forward. From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account -routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an -ordinary mint scoped to its org places the org on its first request. A direct call of the -router's operator place route, or a live placement with the cell's credential, cannot read -the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's -credential is the servers' alone. +routes cannot bring in a new org (`WRONG_CELL`); an ordinary mint scoped to its org places +the org on its first request. A direct call of the router's operator place route, or a live +placement with the cell's credential, cannot read the cell's Postgres and is accepted once +the directory mode is `enforced`; the cell's credential is the servers' alone. ## Orgs attach does not place - **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is - written. One first found in step 4 is skipped and named. Remove such an org by hand - until the stray-org erase command exists, in this order: its bucket objects, from the + written. One first found in step 4 is skipped and named. Remove such an org by hand, in + this order: its bucket objects, from the keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no memory content is left outside erasure; then its rows in every table with an `org` column of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables @@ -128,9 +127,9 @@ credential is the servers' alone. erasure there. `loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, -smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left -unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available -yet. +smallest tables first within `--stray-orgs-budget` (default 60 s), every table the budget +left unchecked and every table it could not read. Remove a stray org's rows by the same +procedure as an over-long org's. ## After attach diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ea9cf77d35..52abea19c6 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -3,26 +3,27 @@ docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and -goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: - -1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; -2. the refusals, before any write: the stores are the registered ones, the session is - direct and marked, the cell's processes (their heartbeats), the roles, settings and - subscriptions that would let an unchecked writer through, the ClickHouse writers, - the table list, durability, attach's own grants; -3. `attach/begin` (the router probes the stores and writes the fingerprints); -4. the mode: `recording`, or `enforced` on a resume; -5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; -6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the - tenancy rows of the orgs found; the triggers again; the pre-commit checks; -7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode - only rises, so every later refusal leaves the cell `enforced` and `attaching`; -8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction - that could have read `recording`, the rescan, the tenancy rows and their placements - (`attach/place`), the final checks and `attach/serving`. - -`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals -and the scan and prints what would be placed, writing nothing. +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. +Steps 1 to 4 are the design's, as cells.md numbers them: + +- the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +- the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, the + table list, durability, attach's own grants; +- `attach/begin` (the router probes the stores and writes the fingerprints); +- step 1, the mode: `recording`, or `enforced` on a resume; +- the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +- in `recording`: step 2, the scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; step 3, + `attach/enforced` and the mode commit, from which on the mode only rises, so every + later refusal leaves the cell `enforced` and `attaching`; +- the durable read of `enforced`, the heartbeats, and step 4: the wait for every + transaction that could have read `recording`, the rescan, the tenancy rows and their + placements (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits in wall time, `--scan-budget` each full scan. `--dry-run` runs +the refusals and the scan and prints what would be placed, writing nothing. """ from __future__ import annotations diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index c79723bbbc..9edb8e275d 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -186,7 +186,10 @@ async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[st try: reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run - reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + reads = [ + (name, WARN, f"not read: {_reason(e)}") + for name in ("cell_tenancy_privilege", "cell_unchecked_writers") + ] return _grades(state, marked, marks_all, defects) + reads @@ -264,6 +267,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: budget = cell_scan.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] + refused: list[str] = [] for i, (_n, store, table) in enumerate(work): found = cell_scan.ScanResult() name = table if store == "postgres" else table.canonical @@ -274,9 +278,15 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: else: for raw in await cell_scan.ch_orgs(ch, table, topology, budget): found.add(name, raw) - except Refused: - unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] - break + except Refused as e: + # The budget spent leaves this table and the larger ones unread; any other + # refusal (a table with no org-led index, a replica that cannot sync) names + # this table alone, and the read goes on. + if budget.left() <= 1: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + refused += [str(r) for r in e.refusals] + continue orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) bad = [m for ms in found.malformed.values() for m in ms] if orgs or bad: @@ -290,13 +300,20 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: or "every org found has a tenancy row", ) ] - if unchecked or problems: + if unchecked: out.append( ( "cell_stray_orgs_unchecked", WARN, - f"not read within --stray-orgs-budget ({budget_s:g} s): " - + ", ".join(unchecked + [str(p) for p in problems]), + f"not read within --stray-orgs-budget ({budget_s:g} s): " + ", ".join(unchecked), + ) + ) + if refused or problems: + out.append( + ( + "cell_stray_orgs_unread", + WARN, + "not read: " + "; ".join(refused + [str(p) for p in problems]), ) ) return out @@ -305,8 +322,9 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row (log-only orgs, names no request can carry, rows an unchecked writer left), smallest - tables first within one budget, naming every table left unchecked. Never a FAIL: the - stray-org erase command removes what it names.""" + tables first within one budget, naming every table left unchecked and every table it + could not read. Never a FAIL: the operator removes what it names by hand + (docs/operations/cell-attach.md).""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client, statement_timeout diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 516dee4cc1..4395138b4d 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -176,6 +176,11 @@ def _function_sql(function: str) -> re.Pattern[str]: ) +def create_statement(table: str, name: str) -> str: + """The CREATE TRIGGER for `name` on `loom.`, as `ensure` writes it.""" + return _create(f"loom.{_ident(table)}", name) + + def _create(qualified: str, name: str) -> str: """The CREATE TRIGGER for `name` on the table `qualified` names.""" if name == TENANCY_TRIGGER: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 9bfe3fe9bf..e0f96f409f 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -668,3 +668,23 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + # An org table with no org-led index: its skip scan refuses, the read goes on. + await pg.execute("CREATE TABLE loom.it_no_org_index (id int PRIMARY KEY, org text)") + await pg.execute("INSERT INTO loom.it_no_org_index VALUES (1, $1)", tag + "unindexed") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + assert tag + "stray" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unread"] + assert grade == "WARN" and "postgres loom.it_no_org_index" in detail + assert "cell_stray_orgs_unchecked" not in lines diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b57a40eb57..857d12dfe0 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -801,3 +801,30 @@ async def run(self: Any) -> int: err = capsys.readouterr().err assert "the mode commit's outcome is unknown: rerun attach" in err assert "stays recording" not in err + + +@pytest.mark.asyncio +async def test_doctor_names_both_catalog_checks_when_the_catalog_is_unread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.cli import doctor_cell + + async def before(_ch: Any) -> str: + return "" + + async def marked() -> tuple[str, bool, list[Any]]: + return "enforced", True, [] + + async def unmarked() -> bool: + return False + + async def unread(_roles: Any) -> Any: + raise OSError("the catalog read failed") + + monkeypatch.setattr(doctor_cell, "_deployment_check", before) + monkeypatch.setattr(doctor_cell, "_marked_session", marked) + monkeypatch.setattr(doctor_cell, "_unmarked_session", unmarked) + monkeypatch.setattr(doctor_cell, "_writer_reads", unread) + lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} + for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): + assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") From 6229357309419724937049dbc62c69d768925439 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 08/25] Attach's closing line names the mode it last read, every DDL try ends inside --wait, and doctor warns on the role it runs as The mode is taken from the first state read, set to unknown only once the router accepted attach/enforced, and set from what the durable read saw; an interrupt prints the mode line and goes on. The trigger DDL's lock bound is taken after the migration lock, from what --wait has left then. An ended mode transaction whose lock does not clear refuses, naming its backend. A busy table's holders come back with ensure_table's answer, and attach says them once per change rather than a boot's warning a try. Attach counts the role it runs as among its own; doctor does not. The dead SCIM recheck and the unused scim_ok parameter are gone. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 18 +- server/cli/cell_attach_beats.py | 4 +- server/cli/cell_attach_pg.py | 71 ++++-- server/cli/doctor_cell.py | 2 +- server/core/cell_triggers.py | 55 +++-- tests/integration/test_cell_attach.py | 71 +++++- .../integration/test_cell_org_triggers_pg.py | 6 +- tests/unit/test_cell_attach.py | 208 +++++++++++++++++- 8 files changed, 368 insertions(+), 67 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 52abea19c6..23449ccff9 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -282,6 +282,8 @@ async def stores_refusals(self) -> list[Refusal]: if recorded is None or str(recorded["deployment_id"]) != expected: out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) state, first = await apg.read_mode(self.lock_conn) + if state in cell.MODES: + self.mode = state # what a refusal from here on leaves the cell at if first != (True, self.name): out.append( Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") @@ -353,7 +355,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: async def catalog_refusals(self) -> list[Refusal]: """The Postgres checks that run again before the mode commit and in step 4.""" - holders = await apg.privilege_holders(self.lock_conn) + holders = await apg.privilege_holders(self.lock_conn, exempt_self=True) self.report["own_roles"] = holders.own out = apg.privilege_refusals(holders, self.args.pg_allow_role) out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) @@ -450,9 +452,9 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg - # From here on the mode may have risen: only the durable read can say. - self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + # From the mode transaction on the mode may have risen: only the durable read can say. + self.mode = "unknown" try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( @@ -491,6 +493,8 @@ async def durable(self) -> int: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break + if read.mode in cell.MODES: + self.mode = read.mode if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -666,9 +670,7 @@ async def precommit(self, found: cell_scan.ScanResult) -> None: ) if out: raise Refused(out) - verdict = await self.check_beats("refuse") - if refusals := beats.final_refusals(verdict): - raise Refused(refusals) + await self.check_beats("refuse") async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: found = await self.scan(topology, listed) @@ -711,6 +713,10 @@ def log(text: str) -> None: log(f"stopped ({type(e).__name__}: {e}); run it again") log(mode_line(run.mode)) return 1 + except BaseException: + # An interrupt: the mode line, then the interrupt as it came. + log(mode_line(run.mode)) + raise run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a9c6e80377..c2a8ed2428 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -180,7 +180,7 @@ def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> N ) -def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: +def final_refusals(verdict: Verdict) -> list[Refusal]: out = [] if not verdict.live and not verdict.booting: out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) @@ -192,7 +192,7 @@ def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: "finish the root-secret rollout first", ) ) - if verdict.scim and not scim_ok: + if verdict.scim: out.append( Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 076dc4a196..9c6e4708d6 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -257,15 +257,16 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: ORDER BY 1, 2 """ -# The roles attach counts as its own: superusers, and a login role that itself owns +# The roles attach counts as its own: superusers, a login role that itself owns # loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the -# triggers, or lower the mode, whatever else it holds). +# triggers, or lower the mode, whatever else it holds), and, for attach only, the role it +# runs as ($2): the operator runs attach, while doctor may run as a CI or monitoring role. _OWN_ROLES = """ SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) -)) OR rolname = current_user +)) OR ($2 AND rolname = current_user) ORDER BY 1 """ @@ -279,9 +280,11 @@ class Holders: others: dict[str, list[str]] -async def privilege_holders(conn: Any) -> Holders: +async def privilege_holders(conn: Any, *, exempt_self: bool) -> Holders: + """The holders of a right `_RIGHTS` counts; `exempt_self` counts the role this session + runs as among the own roles (attach's, not doctor's).""" guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] - own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded, exempt_self)] others: dict[str, list[str]] = {} for r in await conn.fetch(_RIGHTS): if r["role"] not in own: @@ -449,42 +452,49 @@ async def _ensure_table( conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] ) -> Refusal | None: yielded = False + said = "" while True: - lock_ms = 100 + most_ms = 100 holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_ms, yielded = deadlock_ms + 500, True - lock_ms = wait.bound_ms(lock_ms) - result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) - if result == "applied": + most_ms, yielded = deadlock_ms + 500, True + outcome = await _ensure_one(conn, table, most_ms, wait) + if outcome.answer == "applied": return None - if result == "denied": + if outcome.answer == "denied": return Refusal( f"postgres loom.{table}", "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): return _busy_refusal(table, await cell_triggers.holders(conn, table)) - log(f"loom.{table}: its lock is busy; retrying") + # Said again only when the holders change: a retry a second is otherwise silent. + if outcome.holders != said: + said = outcome.holders + log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") -async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: - """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" +async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: + """One try under the migration lock: the migration lock's wait, and then the DDL's lock + wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" from server.core.schema.pg_migrate import _LOCK_KEY migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) if migration_ms <= 0: - return "busy" + return cell_triggers.Outcome("busy") await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return "busy" # a boot migrating: the next try + return cell_triggers.Outcome("busy") # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: + lock_ms = wait.bound_ms(most_ms) + if lock_ms <= 0: + return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) @@ -522,6 +532,9 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: """ _TIMED_OUT = frozenset({"55P03", "57014"}) +# How long attach waits for an ended mode transaction's lock to clear. +_END_ROUNDS = 30 # not a setting: checks of the lock +_END_PAUSE_S = 0.1 # not a setting: the pause between them async def commit_enforced( @@ -585,10 +598,13 @@ async def _mode_try(conn: Any, lock_ms: int) -> bool: async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: - """End an earlier run's mode transaction still in flight and wait until its lock - clears; Refused when its role is a superuser and attach's is not.""" - for _ in range(30): - rows = await conn.fetch(_MODE_HOLDERS) + """End an earlier run's mode transaction still in flight and wait, for up to + `_END_ROUNDS` checks `_END_PAUSE_S` apart, until its lock clears; Refused when its role + is a superuser and attach's is not, or when a backend still holds the lock after the + last check, naming it.""" + rows: list[Any] = [] + for _ in range(_END_ROUNDS): + rows = list(await conn.fetch(_MODE_HOLDERS)) if not rows: return superuser = bool( @@ -608,7 +624,20 @@ async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> N ) await confirm_lock() await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) - await _sleep(0.1) + await _sleep(_END_PAUSE_S) + if not (rows := list(await conn.fetch(_MODE_HOLDERS))): + return + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"still holds loom.tenancy_mode's lock after it was ended " + f"({r['application_name'] or 'no name'}, since {r['xact_start']}): rerun attach " + "once it has gone", + ) + for r in rows + ] + ) # -- the durable read ------------------------------------------------------------------- diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 9edb8e275d..e8a5fc83c1 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -95,7 +95,7 @@ async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, from server.core import pg async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=False) unchecked = await cell_attach_pg.unmarked_refusals(conn, False) roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) return [ diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 4395138b4d..7a1ee5e5b7 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any, Literal +from typing import Any, Literal, NamedTuple from server.core import cell, pg, readiness @@ -95,6 +95,14 @@ # What a table's DDL came to (`ensure_table`, `_run_ddl`). Applied = Literal["applied", "busy", "denied"] + +class Outcome(NamedTuple): + """A table's DDL's answer, and with `busy` the sessions it waited behind.""" + + answer: Applied + holders: str = "" + + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -265,14 +273,15 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: +) -> Outcome: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function passes every row and a boot never creates them. `applied` when the DDL ran or nothing was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, - `denied` when this role cannot alter the table. Raises for a table outside the planned - set, or in another state.""" + `denied` when this role cannot alter the table; a `busy` answer carries the sessions it + waited behind and logs nothing, so the caller says it as often as it retries. Raises + for a table outside the planned set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -284,8 +293,8 @@ async def ensure_table( ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) if not sqls: - return "applied" - return await _run_ddl(conn, table, sqls, lock_timeout, attempts) + return Outcome("applied") + return await _run_ddl(conn, table, sqls, lock_timeout, attempts, warn=False) def _table_statements( @@ -308,11 +317,14 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: + *, + warn: bool = True, +) -> Outcome: """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the - two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: - a table this role cannot alter), which a boot serves past; any other error fails it, as - a failed read of the state does.""" + two causes cells.md names kept it from applying (`busy`: its lock not taken, with the + sessions it waited behind; `denied`: a table this role cannot alter), which a boot + serves past; any other error fails it, as a failed read of the state does. `warn` logs + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" import asyncpg for attempt in range(attempts): @@ -321,25 +333,28 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return "applied" + return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return "denied" + return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue - log.warning( - "cell triggers: not applied on loom.%s (%s): %s; %s", - table, - e, - "; ".join(sqls), - await _holders(conn, table), - ) - return "busy" + held = await _holders(conn, table) + if warn: + log.warning( + "cell triggers: not applied on loom.%s (%s): %s; %s", + table, + e, + "; ".join(sqls), + held, + ) + return Outcome("busy", held) + return Outcome("busy") async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index e0f96f409f..d77592d1dc 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -532,7 +532,7 @@ async def test_each_right_that_makes_an_org_held_is_named( await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) for sql in grant.format(r=role, owner=owner).split("; "): await conn.execute(sql) - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) assert owner in holders.own assert any(why in w for w in holders.others.get(role, [])), holders.others finally: @@ -550,7 +550,7 @@ async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> N try: async with pg.dedicated() as conn: await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] finally: @@ -631,8 +631,40 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} grade, detail = lines["cell_stray_orgs"] assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail - tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} - assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( + cell_stores, monkeypatch +) -> None: + from server.cli import cell_scan, doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("CREATE TABLE loom.it_small (org text PRIMARY KEY)") + await pg.execute("CREATE TABLE loom.it_large (org text PRIMARY KEY)") + await pg.execute("INSERT INTO loom.it_small VALUES ($1)", tag + "small") + await pg.execute( + "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" + ) + await pg.execute("ANALYZE loom.it_small, loom.it_large") + # The budget runs out on the largest table, as it does on a real cell when that table's + # read is what is too slow: every smaller one was read first. + real, read = cell_scan.pg_orgs, [] + + async def pg_orgs(table: str, budget: Any) -> list[str]: + read.append(table) + if table == "loom.it_large": + await asyncio.sleep(budget.left() + 0.1) + return await real(table, budget) + + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} + assert read[-1] == "loom.it_large" and "loom.it_small" in read + assert tag + "small" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unchecked"] + assert grade == "WARN" and detail.endswith(": loom.it_large") @pytest.mark.asyncio @@ -657,10 +689,10 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking assert any("loom.scope_generations" in r for r in lacking) # The trigger pass tells the cause apart: denied, not a busy lock. - assert ( - await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - == "denied" + outcome = await cell_triggers.ensure_table( + conn, "scope_generations", recording_first=True ) + assert outcome.answer == "denied" await conn.execute("RESET ROLE") assert await _mode() == "recording" finally: @@ -688,3 +720,28 @@ async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_store grade, detail = lines["cell_stray_orgs_unread"] assert grade == "WARN" and "postgres loom.it_no_org_index" in detail assert "cell_stray_orgs_unchecked" not in lines + + +@pytest.mark.asyncio +async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_ci_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA loom TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"SET ROLE {role}") + as_attach = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + as_doctor = await cell_attach_pg.privilege_holders(conn, exempt_self=False) + await conn.execute("RESET ROLE") + # Attach runs as the operator's role; a CI role doctor runs as is warned about. + assert role in as_attach.own and role not in as_attach.others + assert role not in as_doctor.own + assert "may insert into loom.tenancy" in as_doctor.others[role] + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index b2fb66a467..c219d05777 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -722,15 +722,15 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") ensure = cell_triggers.ensure_table - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" - assert await ensure(conn, "tenancy", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" + assert (await ensure(conn, "tenancy", recording_first=True)).answer == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 857d12dfe0..b3db51ee02 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -268,8 +268,8 @@ async def planned(_c: Any) -> list[str]: async def holders(_c: Any, _t: str) -> list[Any]: return [] - async def ensure_table(*_a: Any, **_kw: Any) -> str: - return "denied" + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("denied") monkeypatch.setattr(cell_triggers, "planned_tables", planned) monkeypatch.setattr(cell_triggers, "holders", holders) @@ -280,14 +280,22 @@ async def ensure_table(*_a: Any, **_kw: Any) -> str: assert refusal.subject == "postgres loom.scope_generations" assert "cannot alter it" in refusal.reason and wait.spent < 1 - async def busy(*_a: Any, **_kw: Any) -> str: + holders = iter(["pid 7", "pid 7", "pid 7", "pid 8"] + ["pid 8"] * 100) + + async def busy(*_a: Any, **_kw: Any) -> Any: clock.now += 0.1 - return "busy" + return cell_triggers.Outcome("busy", next(holders)) monkeypatch.setattr(cell_triggers, "ensure_table", busy) - wait = apg.Wait(3, clock) - (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) - assert "not taken within --wait" in refusal.reason and wait.spent <= 3 + wait = apg.Wait(10, clock) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) + assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # Said once per change of holders, not once a try. + assert logged == [ + "loom.scope_generations: its lock is busy (pid 7); retrying", + "loom.scope_generations: its lock is busy (pid 8); retrying", + ] # -- the rescan's watch ---------------------------------------------------------------- @@ -828,3 +836,189 @@ async def unread(_roles: Any) -> Any: lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") + + +@pytest.mark.asyncio +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: + """A migration lock that took the rest of --wait leaves no DDL try.""" + + def migration_lock(*_a: Any) -> str: + clock.now += 2.0 # the rest of --wait + return "OK" + + conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) + wait = apg.Wait(2, clock) + with wait.counting(): + outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) + assert outcome.answer == "busy" + assert not any( + "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + ) + + +@pytest.mark.asyncio +async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: + held = [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}] + conn = FakeConn( + [ + ("l.mode = 'ExclusiveLock'", held), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused) as e: + await apg.end_mode_transactions(conn, confirm) + (refusal,) = e.value.refusals + assert refusal.subject == "postgres backend 5" + assert "still holds loom.tenancy_mode's lock" in refusal.reason + + +def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Run, "run", body) + + +@pytest.mark.asyncio +async def test_a_router_refusal_of_attach_enforced_leaves_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: + raise Refused([Refusal("router", f"refused {path}")]) + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + await self.commit_mode() + return 0 + + monkeypatch.setattr(cell_attach.Router, "call", refuse) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_a_durable_read_that_sees_recording_says_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + async def read() -> apg.Durable: + return apg.Durable(mode="recording", first=(True, "first"), xid=1) + + async def run(self: Any) -> int: + self.mode = "unknown" # a mode commit whose outcome was lost + await self.durable() + return 0 + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + monkeypatch.setattr(apg, "durable_read", read) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_an_enforced_resume_refused_early_says_enforced( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def recorded(_ch: Any) -> dict[str, Any]: + return {"deployment_id": "other"} + + async def health(_self: Any, _host: str) -> str: + return "dep-1" + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return "enforced", (True, "first") + + async def run(self: Any) -> int: + self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} + self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) + if refusals := await self.stores_refusals(): + raise Refused(refusals) + return 0 + + monkeypatch.setattr(pg_profile, "recorded_authority", recorded) + monkeypatch.setattr(cell_attach.Router, "health", health) + monkeypatch.setattr(apg, "read_mode", read_mode) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + err = capsys.readouterr().err.splitlines() + assert "clickhouse: records another deployment id" in err[0] + assert err[-1] == ( + "loom cell attach: the cell stays enforced and attaching: rerun attach to resume" + ) + + +@pytest.mark.asyncio +async def test_an_interrupt_prints_the_mode_line_and_goes_on( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def run(self: Any) -> int: + self.mode = "unknown" + raise KeyboardInterrupt + + _stub_attach(monkeypatch, run) + with pytest.raises(KeyboardInterrupt): + await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) + assert "the mode commit's outcome is unknown" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_the_stray_read_names_a_refused_table_and_reads_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.cli import cell_ch, doctor_cell + from server.core import pg + + async def topology(_ch: Any) -> Any: + return cell_ch.Topology("", 0) + + async def no_tables(*_a: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + async def sizes(*_a: Any) -> dict[str, int]: + return {} + + pg_rows = [{"name": "loom.a_unindexed", "n": 1}, {"name": "loom.b_data", "n": 2}] + conn = FakeConn([("FROM loom.tenancy", []), ("pg_class c", pg_rows)]) + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + async def pg_orgs(table: str, _budget: Any) -> list[str]: + if table == "loom.a_unindexed": + raise Refused([Refusal(f"postgres {table}", "has no plain btree index led by org")]) + return ["stray-org"] + + monkeypatch.setattr(cell_ch, "read_topology", topology) + monkeypatch.setattr(cell_scan, "list_clickhouse", no_tables) + monkeypatch.setattr(doctor_cell, "_table_sizes", sizes) + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(None, 60)} + assert lines["cell_stray_orgs"] == ("WARN", "postgres loom.b_data: stray-org") + assert lines["cell_stray_orgs_unread"] == ( + "WARN", + "not read: postgres loom.a_unindexed: has no plain btree index led by org", + ) + assert "cell_stray_orgs_unchecked" not in lines From a64cbb28b5ef130e72f67ccad16657b8edeca930 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 09/25] The attach docs say which role attach exempts, and the trigger gate's figures say what they establish cells.md and the runbook state that attach exempts the role it runs as and doctor does not. results.md gives the marked batch's per-round changes and their mean, says the delivery-pass gate is met on the mean but not established by three rounds alone, and that the memory.set p99 gate is unresolved locally. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 +++++---- docs/benchmarks/results.md | 19 +++++++++++-------- docs/operations/cell-attach.md | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 3ea7ddab80..246da73b77 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1007,10 +1007,11 @@ It refuses, before any write, naming each: without one, or a role that may not write the durable read's logical message; a role of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with `max_prepared_transactions` above 0; -- **the tenancy-privilege check**: a login role, other than attach's own (superusers, - and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and - those named with `--pg-allow-role`, that holds or reaches through any membership a right - to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or +- **the tenancy-privilege check**: a login role, other than attach's own (the role attach + runs as, which the operator runs it with; superusers; and a role that itself owns + `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and those named with + `--pg-allow-role`, that holds or reaches through any membership a right to make an org + held or lower the mode: `INSERT` on any column of `loom.tenancy` or `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system one (a function there can shadow one the servers call unqualified on their marked diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 81e099d3c8..4814bd7f5f 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1241,14 +1241,17 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url | 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | | 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | -The marked batch, which is what a delivery pass of a cell's own processes writes, shows no -cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. -The `memory.set` figures move by more between rounds of one configuration than any trigger -cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms -here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no -consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the -host's noise, and the statement-level figures above bound the marked cost at under a -microsecond a row. An unmarked session pays about 11 µs a row, as above. +The marked batch is what a delivery pass of a cell's own processes writes. On minus off +within each round it moved +1.3, −1.9 and −2.1 ms (+13.4%, −17.1% and −19.8%); the means are +9.6 ms on and 10.5 ms off, −0.9 ms (−8.6%). The delivery-pass gate (at most 10% slower) is +met on the mean, but one round of three was 13% slower and three rounds give no paired bound +(methodology's paired bootstrap is for per-item labels), so this run does not establish the +gate on its own; the statement-level figures above do, at under a microsecond a row against +about 10 µs a row of insert. The `memory.set` figures move by more between rounds of one +configuration than any trigger cost: on this loaded host the p99 differences within a round +(+214.8, +47.0, +24.3 ms here, and −71.7, −10.0, −42.4 ms in an earlier run of the same +measurement) have no consistent sign: the `memory.set` p99 gate (a rise of at most 10 ms) is +unresolved locally. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index c51541d06d..481b9a95ef 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -135,10 +135,11 @@ procedure as an over-long org's. `loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an -org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or -a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a -writer added after `serving` passes attach's checks, which ran once; doctor is how it is -found. +org held or lower the mode (`cell_tenancy_privilege`). Attach exempts the role it runs as, +the operator's; doctor does not exempt the role it runs as, so a CI or monitoring role +holding such a right is named. It warns too on a role or database setting or a subscription +that passes an unchecked writer (`cell_unchecked_writers`). A grant or a writer added after +`serving` passes attach's checks, which ran once; doctor is how it is found. ## Related From 84e64603095fe8c2bf47cab47998209ce1c08fa6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 10/25] Every attach try runs once whatever --wait has left, the mode reads unknown only while a try's UPDATE is out, and a busy migration lock is named Each trigger and mode-commit try gets a lock wait of at least TRY_FLOOR_MS before --wait is judged spent, so --wait 0 is one try of each wait with no retry, and an idle cell attaches with it. The run's mode turns unknown just before a try sends its UPDATE and back to recording when that try rolls back. A trigger try that finds the migration lock busy names that lock's holder, in the retry line and in the refusal. The step-4 watch task is awaited after its cancel. The ATTACH_LOCK_KEY comment says who holds the lock. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 24 +++- server/cli/cell_attach_pg.py | 74 ++++++++--- server/core/cell.py | 6 +- tests/integration/test_cell_attach.py | 17 ++- tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_attach.py | 176 ++++++++++++++++++++++++-- 7 files changed, 260 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 23449ccff9..8ed70e2d37 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -30,6 +30,7 @@ import argparse import asyncio +import contextlib import json import sys import time @@ -453,16 +454,22 @@ async def commit_mode(self) -> None: from server.core import pg await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) - # From the mode transaction on the mode may have risen: only the durable read can say. - self.mode = "unknown" + + def mark(mode: str) -> None: + # `unknown` from a try's UPDATE on: only the durable read can say whether it + # landed; `recording` again when that try rolled back. + self.mode = mode + try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark ) except Refused: raise - except Exception as e: # noqa: BLE001 — the durable read says whether it landed + except Exception as e: + if self.mode != "unknown": + raise # no try was in flight: nothing to decide self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.end_mode_transactions(conn, self.confirm_lock) @@ -573,6 +580,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: raise Refused(writers) finally: task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task await self.final_checks(watch) row = await self.router.call("") current, nxt = self.own_fp @@ -757,7 +766,12 @@ async def _run(ch: Any) -> int: def add_attach_parser(csub: Any) -> None: p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") - p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--wait", + type=int, + default=_WAIT_S, + help="Seconds of wall time the waits share; 0 runs one try of each, with no retry", + ) p.add_argument( "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 9c6e4708d6..0ca9fefcc1 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -24,6 +24,8 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval +# The least lock wait a try gets, however little of --wait is left. +TRY_FLOOR_MS = 100 # not a setting: one try's floor # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound @@ -67,8 +69,10 @@ def counting(self) -> Iterator[None]: self._since = None def bound_ms(self, most_ms: int) -> int: - """A lock timeout for one try: `most_ms`, or what is left when that is less.""" - return min(most_ms, int(self.left() * 1000)) + """A lock timeout for one try: `most_ms`, or what is left when that is less, and + never under `TRY_FLOOR_MS`: every try runs once, so `--wait 0` is one try of each + wait with no retry, and only after a try is the wait judged spent.""" + return max(TRY_FLOOR_MS, min(most_ms, int(self.left() * 1000))) async def pause(self, seconds: float = POLL_S) -> bool: """Wait `seconds` (at most what is left); False once none is left.""" @@ -89,6 +93,13 @@ def _split(key: int) -> str: ) +# Who holds an advisory lock (`where` from `_split`), never a query's text. +_ADVISORY_HOLDER = """ +SELECT l.pid, a.application_name, a.backend_start +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {where} AND l.granted +""" + _LOCK_HOLDER = f""" SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid @@ -467,6 +478,11 @@ async def _ensure_table( "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): + if outcome.holders.startswith(_MIGRATION_HELD): + return Refusal( + f"postgres loom.{table}", + f"its triggers were not created within --wait: {outcome.holders}", + ) return _busy_refusal(table, await cell_triggers.holders(conn, table)) # Said again only when the holders change: a retry a second is otherwise silent. if outcome.holders != said: @@ -474,29 +490,45 @@ async def _ensure_table( log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") +_MIGRATION_HELD = "the migration lock" + + +async def _migration_holder(conn: Any, key: int) -> str: + """`the migration lock (held by …)`, naming the session that holds it.""" + try: + rows = await conn.fetch(_ADVISORY_HOLDER.format(where=_split(key))) + except Exception as e: # noqa: BLE001 — naming the holder is best effort + return f"{_MIGRATION_HELD} (its holder not read: {type(e).__name__})" + held = "; ".join( + f"pid {r['pid']}, {r['application_name'] or 'no name'}, since {r['backend_start']}" + for r in rows + ) + return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" + + async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock - wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" + wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the + try floor. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) - if migration_ms <= 0: - return cell_triggers.Outcome("busy") - await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return cell_triggers.Outcome("busy") # a boot migrating: the next try + # A boot migrating: the next try. + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) raise finally: await conn.execute("RESET lock_timeout") try: - lock_ms = wait.bound_ms(most_ms) - if lock_ms <= 0: - return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, ) finally: await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) @@ -543,21 +575,29 @@ async def commit_enforced( ready: Callable[[], Any], wait: Wait, log: Callable[[str], None], + mark: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before each try (False skips the try while a process boots). After a failed try the in-flight - mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; - whether the commit landed is the durable read's to say.""" + mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try + sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` + runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - lock_ms = wait.bound_ms(_MODE_LOCK_MS) - if lock_ms > 0 and await ready(): - if await _mode_try(conn, lock_ms): + if await ready(): + mark("unknown") + try: + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + except Refused: + mark("recording") # the UPDATE's transaction rolled back + raise + if done: return + mark("recording") log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..37e20f99cb 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -62,9 +62,9 @@ # Postgres's btree entry limit (about 2.7 KB), so every org a cell records has a # placement the directory can index. ORG_MAX_BYTES = 2048 # not a setting: cells and the router judge one limit -# The advisory lock on a cell's Postgres that attach will hold, and prepare and update once -# they take it, and that the troubleshooting page's full-state cleanup takes before the -# migration lock. Positive: the page's holder query splits it into two `oid` halves. +# The advisory lock on a cell's Postgres that attach and prepare hold for their run, and that +# the troubleshooting page's full-state cleanup takes before the migration lock. Positive: +# the page's holder query splits it into two `oid` halves. ATTACH_LOCK_KEY = 0x6C6F6F6D_61746368 # "loomatch" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index d77592d1dc..65064ab365 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -482,7 +482,9 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) - await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await cell_attach_pg.commit_enforced( + conn, ok, ok, cell_attach_pg.Wait(20), logged.append, lambda _m: None + ) await task assert await _mode() == "enforced" assert logged == ["the mode's lock is busy; retrying"] @@ -745,3 +747,16 @@ async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_store await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_wait_0_attaches_an_idle_cell(cell_stores, capsys) -> None: + """--wait 0 is one try of each wait, no retry: an idle cell still reaches serving.""" + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "w0" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + capsys.readouterr() + assert await _attach(stores, network, wait=0) == 0, capsys.readouterr().err + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert org in await _placed() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 3e88dbafca..4e45db4285 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -461,7 +461,7 @@ ], "loom cell attach": [ "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 3d597c1e03..a31a862106 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -13032,7 +13032,7 @@ "nargs": null, "type": "builtins:int", "required": false, - "help": "Seconds the waits share", + "help": "Seconds of wall time the waits share; 0 runs one try of each, with no retry", "metavar": null }, { diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b3db51ee02..429c2fcfb7 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -250,7 +250,7 @@ async def ready() -> bool: wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None, lambda _m: None) assert clock.now - 100 <= 4 and wait.spent <= 4 # A 2 s try, a 1 s pause, then a try that waits only the second left. assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] @@ -290,7 +290,9 @@ async def busy(*_a: Any, **_kw: Any) -> Any: wait = apg.Wait(10, clock) logged: list[str] = [] (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) - assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # The last try ran with the floor's lock wait, after --wait was all but spent. + assert "not taken within --wait" in refusal.reason + assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 # Said once per change of holders, not once a try. assert logged == [ "loom.scope_generations: its lock is busy (pid 7); retrying", @@ -654,7 +656,9 @@ async def ready() -> bool: return True logged: list[str] = [] - await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(10, clock), logged.append, lambda _m: None + ) assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 @@ -692,7 +696,9 @@ async def booting() -> bool: return False with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + await apg.commit_enforced( + conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + ) assert not any("LOCK TABLE" in s for s in conn.executed) @@ -789,8 +795,8 @@ async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outc async def dedicated(**_kw: Any) -> Any: yield None - async def landed(*_a: Any) -> None: - pass # the commit landed; its answer may not have + async def landed(*a: Any) -> None: + a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} @@ -839,22 +845,166 @@ async def unread(_roles: Any) -> Any: @pytest.mark.asyncio -async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: - """A migration lock that took the rest of --wait leaves no DDL try.""" +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + """A migration lock that took the rest of --wait leaves the DDL its floor: one try, never + the full 100 ms when less is left, never none.""" + from server.core import cell_triggers def migration_lock(*_a: Any) -> str: - clock.now += 2.0 # the rest of --wait + clock.now += 1.95 # all but 50 ms of --wait return "OK" + asked: list[str] = [] + + async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: + asked.append(lock_timeout) + return cell_triggers.Outcome("applied") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) wait = apg.Wait(2, clock) with wait.counting(): - outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) - assert outcome.answer == "busy" - assert not any( - "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + outcome = await apg._ensure_one(conn, "scope_generations", 500, wait) + assert outcome.answer == "applied" + assert asked == [f"{apg.TRY_FLOOR_MS}ms"] + + +@pytest.mark.asyncio +async def test_a_busy_migration_lock_is_named_as_such( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + class _Busy(Exception): + sqlstate = "55P03" + + holder = [{"pid": 42, "application_name": "loom", "backend_start": "t0"}] + conn = FakeConn( + [ + ("pg_advisory_lock", _Busy()), + ("l.locktype = 'advisory'", holder), + ("deadlock_timeout", 1000), + ] ) + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, apg.Wait(2, clock), logged.append) + named = "the migration lock (held by pid 42, loom, since t0)" + assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] + assert refusal.reason == f"its triggers were not created within --wait: {named}" + + +@pytest.mark.asyncio +async def test_wait_0_runs_one_try_of_each_wait( + clock: _Clock, monkeypatch: pytest.MonkeyPatch +) -> None: + """On an idle database --wait 0 commits the mode and creates the triggers in one try + each, and the heartbeats pass when nothing boots.""" + from server.core import cell_triggers + + conn = FakeConn( + [ + ("LOCK TABLE", "LOCK TABLE"), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + marks: list[str] = [] + + async def ok() -> bool: + return True + + async def nothing() -> None: + pass + + await apg.commit_enforced(conn, nothing, ok, apg.Wait(0, clock), lambda _t: None, marks.append) + assert marks == ["unknown"] + assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def applied(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("applied") + + async def no_defects(_c: Any) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + monkeypatch.setattr(cell_triggers, "ensure_table", applied) + monkeypatch.setattr(cell_triggers, "missing", no_defects) + ddl = FakeConn([("deadlock_timeout", 1000)]) + assert await apg.ensure_triggers(ddl, apg.Wait(0, clock), lambda _t: None) == [] + + async def read() -> list[beats.Beat]: + return [_beat("a")] + + assert (await beats.settle(read, _judge, apg.Wait(0, clock))).live == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("refused_before_any_try", [True, False]) +async def test_a_mode_commit_refused_without_a_landed_try_says_recording( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + clock: _Clock, + refused_before_any_try: bool, +) -> None: + """The closing line says recording when no try sent its UPDATE (the heartbeats refused + before one) and when every try timed out and rolled back.""" + + def lock(*_a: Any) -> str: + raise _Timeout() + + conn = FakeConn([("LOCK TABLE", lock), ("l.mode = 'ExclusiveLock'", [])]) + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def ready(_self: Any) -> bool: + if refused_before_any_try: + raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) + return True + + async def confirm(_self: Any) -> None: + pass + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.wait = apg.Wait(2, clock) + await self.commit_mode() + return 0 + + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) + monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) + monkeypatch.setattr(cell_attach.Run, "run", run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + @pytest.mark.asyncio async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: From 5615d41a038625da621b7d25431284b6e8acdb12 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 11/25] The attach docs say what --wait 0 does, and cells.md keeps each code span on one line inside its list items Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 185 +++++++++++++++++---------------- docs/operations/cell-attach.md | 27 ++--- 2 files changed, 108 insertions(+), 104 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 246da73b77..53495c7a1d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1099,12 +1099,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): `enforced`; then `attach/serving`. `--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try -included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each -full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints -what would be placed and skipped and the scan's time, and writes nothing; the router's -checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: -`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's -Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. +included and never longer than what is left, but for a floor of 100 ms: every try runs once +before the wait is judged spent, so `--wait 0` is one try of each wait with no retry; +`--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. +`--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the +scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a +`serving` row attach reads the cell's mode: `enforced` is done (exit 0), and any other mode +refuses: the directory and the cell's Postgres disagree, as after a restore of one of them, +and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1487,58 +1489,58 @@ An integration test with two cells and a router in CI's Postgres job: between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token - revocation) runs, also with Postgres down on a deployment never prepared, and a `loom -cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres - down; after `loom cell credential --cell --next` both credentials place and read - state, `--finish` refuses the old credential, for a cell finished before its pods hold the - new one the router refuses its calls with 401, its request answers 503 writing nothing, - and it recovers by recording the old hash again, `--revert` drops a pending `next`, which - is then refused, and the old credential keeps placing, and a second `--next` while one is - pending and a hash another cell records as current or next are refused; `loom cell limit` - and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is - unavailable; concurrent placements at one under `placements_max` across two router - replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set - refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a - later cell, and on a first cell in `recording` runs and records the org; attach with - `placements_max` below the orgs the first cell holds places them all, and a later cell's - attach past its quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step 2 - and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a - process whose cache still reads `recording` sends a new org's request to the placement - call, which a closed cell answers 503; a CLI command whose insert finds `enforced` - refuses, naming the org, and makes no placement call; a refusal after step 3's write - leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; - each ordering of attach's write of the mode with a recording insert holds as stated above - (running, starting during the write, a stream of overlapping inserts each shorter than a - try, with attach setting `enforced` within a try or two, held across several tries, held - past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut - off by its timeout, and a commit whose connection is lost while it waits for the standby, - which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran - before `enforced` committed, on attach's role or a second role, holds step 4 until it ends - and its write of an unheld org is then placed, one whose first statement ran after is - refused by the trigger, a transaction that starts after attach takes the lock but before - its commit is visible holds step 4, a transaction opened in `recording` that outlives one - run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom - table does not hold step 4 and a background worker with a transaction open since - `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` - raised during the wait) makes step 4 refuse at once, naming it, with the mode still - `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write - transaction open in a second database on the same server holds step 4 and is named, a long - read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on - does not hold it, nor does a base backup running across it, a write held open on a - `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a long - logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past - `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first - cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 - refuses before step 1 with the mode still `recording`, while a later cell's attach does - not require either; a missing or disabled org trigger is refused by attach and named by - `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted - is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset - is unready and runs no worker, and a CLI command declaring an org with it unset is - refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` - cell is refused, also under `session_replication_role = replica`, and a missing or - disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + revocation) runs, also with Postgres down on a deployment never prepared, and a + `loom cell` store command refuses; `loom cell limit --closed` succeeds with the cell's + Postgres down; after `loom cell credential --cell --next` both credentials place + and read state, `--finish` refuses the old credential, for a cell finished before its pods + hold the new one the router refuses its calls with 401, its request answers 503 writing + nothing, and it recovers by recording the old hash again, `--revert` drops a pending + `next`, which is then refused, and the old credential keeps placing, and a second `--next` + while one is pending and a hash another cell records as current or next are refused; + `loom cell limit` and `loom cell credential --next`, `--finish` and `--revert` refuse + while the directory is unavailable; concurrent placements at one under `placements_max` + across two router replicas write one; a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a first cell once it is `enforced` + (`attaching` included) and on a later cell, and on a first cell in `recording` runs and + records the org; attach with `placements_max` below the orgs the first cell holds places + them all, and a later cell's attach past its quota or closed refuses, naming the org; a + first cell with `--closed` and `placements_max` set before step 2 attaches, and an org a + request records between step 2 and `enforced` is placed by step 4 with no 503; after step + 3's write of the cell's mode, a process whose cache still reads `recording` sends a new + org's request to the placement call, which a closed cell answers 503; a CLI command whose + insert finds `enforced` refuses, naming the org, and makes no placement call; a refusal + after step 3's write leaves the cell `enforced` and `attaching`, where a new org is placed + through the gate; each ordering of attach's write of the mode with a recording insert + holds as stated above (running, starting during the write, a stream of overlapping inserts + each shorter than a try, with attach setting `enforced` within a try or two, held across + several tries, held past `--wait` (attach refuses, the cell still `recording`), a stalled + attach, a commit cut off by its timeout, and a commit whose connection is lost while it + waits for the standby, which the durable read decides); a `REPEATABLE READ` transaction + whose first statement ran before `enforced` committed, on attach's role or a second role, + holds step 4 until it ends and its write of an unheld org is then placed, one whose first + statement ran after is refused by the trigger, a transaction that starts after attach + takes the lock but before its commit is visible holds step 4, a transaction opened in + `recording` that outlives one run's `--wait` holds the rerun's step 4 after a refusal, an + autovacuum worker on a loom table does not hold step 4 and a background worker with a + transaction open since `recording` does, a prepared transaction (after a restart with + `max_prepared_transactions` raised during the wait) makes step 4 refuse at once, naming + it, with the mode still `enforced` and a rerun refused until it is resolved and the + setting is back to 0, a write transaction open in a second database on the same server + holds step 4 and is named, a long read on the synchronous standby, or on a second standby, + with `hot_standby_feedback` on does not hold it, nor does a base backup running across it, + a write held open on a `replication=database` connection holds step 4, and so does a + `REPEATABLE READ` transaction on one, opened in `recording` and writing nothing before + `enforced`, a long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's + database past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and + the first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` + above 0 refuses before step 1 with the mode still `recording`, while a later cell's attach + does not require either; a missing or disabled org trigger is refused by attach and named + by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was + deleted is refused and the state named by `loom doctor`; on that cell a pod with + `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an org with + it unset is refused, as a pod is; updating, deleting or truncating the `first` row on an + `enforced` cell is refused, also under `session_replication_role = replica`, and a missing + or disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer request and runs no worker pass, and serves again after a rerun of `loom cell prepare` writes the row back while it runs; a cell the directory registers with no state, tenancy @@ -1548,12 +1550,12 @@ cell` store command refuses; `loom cell limit --closed` succeeds with the cell's rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, has the function refuse an unheld org (a boot creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a - Postgres-only restore of an attached first cell to before its prepare is named by `loom -doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell -update` on an attached cell, with processes on both values and `loom.cell_self` lost, - writes both values back and keeps both sets of processes serving, and one before attach - refuses the two configurations; a prepare rerun on an attached cell whose `current` is - missing and whose pods' configuration differs from the directory row refuses, and one + Postgres-only restore of an attached first cell to before its prepare is named by + `loom doctor` and closed by `loom cell restore`; a prepare rerun during a pending + `loom cell update` on an attached cell, with processes on both values and `loom.cell_self` + lost, writes both values back and keeps both sets of processes serving, and one before + attach refuses the two configurations; a prepare rerun on an attached cell whose `current` + is missing and whose pods' configuration differs from the directory row refuses, and one before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a credential hash that is neither the row's `current` nor its `next` is refused; a prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a @@ -1561,33 +1563,34 @@ update` on an attached cell, with processes on both values and `loom.cell_self` creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the row back; a later cell whose mode row is set to `recording` records nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as - in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom -cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` - set refuses, naming it, and the router refuses its state read; a build N-1 process's - `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed - and write no tenancy row, and on the first cell in `recording` a boot creates no org - trigger, and one created by hand reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), an - `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one - that hits an existing row of it is not, a marked session's every write passes, and the - triggers on an `enforced` cell cover every table erasure classifies with an `org` column - but the three exclusions; and an org-keyed write while attach holds the mode table's lock - returns at once, with new-org requests answering within their deadline and mode-cache - reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` - with its directory row `enforced`, and the rerun runs step 2 again; a - `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom -doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with - the directory and the stores unreachable; prepare and `--next` refuse an argument that is - not 64 lowercase hex characters, the credential and its bare 128-character secret - included, storing nothing; orgs a first cell records in `recording` with no data are - listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at - each router replica, `loom doctor` names the cell as one with no quota, and a call refused - past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its - retry at that replica, with no other call of the cell admitted between, is admitted; a - cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or - CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the - authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep - working through the cutover, a logout through the shared host revokes its refresh token; + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, + `loom cell prepare` names the row, a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses its state read; a + build N-1 process's `memory.set` and delivery pass on a never-prepared deployment after + N's migration succeed and write no tenancy row, and on the first cell in `recording` a + boot creates no org trigger, and one created by hand reads the state and looks nothing up, + a write under `session_replication_role = replica` for an unheld org is skipped (origin + triggers), an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is + refused and one that hits an existing row of it is not, a marked session's every write + passes, and the triggers on an `enforced` cell cover every table erasure classifies with + an `org` column but the three exclusions; and an org-keyed write while attach holds the + mode table's lock returns at once, with new-org requests answering within their deadline + and mode-cache reads returning at once throughout; a refusal at step 3's lock leaves the + cell `recording` with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by + `loom doctor`; `loom cell credential new` prints a credential of that form and its + SHA-256, with the directory and the stores unreachable; prepare and `--next` refuse an + argument that is not 64 lowercase hex characters, the credential and its bare + 128-character secret included, storing nothing; orgs a first cell records in `recording` + with no data are listed by `loom doctor`; a placement flood with no quota set is held to + the rate limit at each router replica, `loom doctor` names the cell as one with no quota, + and a call refused past it at one replica answers 503 with a `Retry-After` of at most 60 + s, after which its retry at that replica, with no other call of the cell admitted between, + is admitted; a cell's first prepare, with no directory row yet, passes the deployment-id + check; a pod or CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is + refused by the authority check; the first cell's existing tokens, refresh tokens and OAuth + clients keep working through the cutover, a logout through the shared host revokes its + refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 481b9a95ef..b713aefe0d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -59,14 +59,14 @@ Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` o `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` -on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role -of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the -writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, -`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT -QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings -profile or quota capping the rows or bytes it reads, and the insert into -`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the -bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role of +`loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the writer +check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, +`SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and +`READ ON REMOTE`), no settings profile or quota capping the rows or bytes it reads, and the +insert into `loom._migration_lock` that writes the probe nonce. The CLI needs write access +to the bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. ## Run it @@ -80,13 +80,14 @@ loom cell attach --cell \ `--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs per source table, the ones it would skip, and whether the scan fit in half of `--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, -the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object -and replica that may write the configured databases with `--ch-allow-user`, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object and +replica that may write the configured databases with `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each -list is audited by the router. `--wait` (default 600 s) bounds the waits together; -`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of -it, since step 4 reads the same tables again. +list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; +every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs +one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, +and step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it From b0f694b7e52bb17475cedb5f005de18d353dc435 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:05:39 +0000 Subject: [PATCH 12/25] Bound cell attach's mode commit and name booting processes Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +- docs/operations/cell-attach.md | 6 +- server/cli/cell_attach.py | 4 +- server/cli/cell_attach_pg.py | 34 +++++--- tests/integration/test_cell_attach.py | 4 +- tests/unit/test_cell_attach.py | 111 +++++++++++++++++++++++--- 6 files changed, 134 insertions(+), 31 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 53495c7a1d..f32815b635 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1065,8 +1065,10 @@ Then, in order (the step numbers are the ones the rest of this page uses): backed out. - **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock (only + cheap heartbeat refusals run before each try (a booting process skips the try). Each try, + including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and + a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell `enforced` and `attaching`, and a rerun resumes from the durable read. diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index b713aefe0d..ec6b6354ec 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -86,8 +86,10 @@ replica that may write the configured databases with `--ch-allow-user`, Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs -one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, -and step 2 must take at most half of it, since step 4 reads the same tables again. +one try of each wait with no retry. The final heartbeat checks get at least two heartbeats +plus the router timeout, with one extension for a process first seen booting; this floor +is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and +step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8ed70e2d37..6ef819796a 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -442,13 +442,13 @@ async def triggers(self) -> None: # -- step 3 ------------------------------------------------------------------------ - async def ready_for_commit(self) -> bool: + async def ready_for_commit(self) -> list[str]: """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, a booting process skips the try.""" verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) if verdict.refusals: raise Refused(verdict.refusals) - return not verdict.booting + return verdict.booting async def commit_mode(self) -> None: from server.core import pg diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 0ca9fefcc1..60196363af 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -12,7 +12,7 @@ import asyncio import contextlib import time -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -29,9 +29,13 @@ # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# A trigger try yields quickly to active writers. +_TRIGGER_LOCK_MS = 100 # not a setting: one try's bound # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait +# COMMIT's standby wait is outside Postgres's statement timeout. +_MODE_STANDBY_S = 5.0 # not a setting: one try's standby allowance # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} @@ -465,7 +469,7 @@ async def _ensure_table( yielded = False said = "" while True: - most_ms = 100 + most_ms = _TRIGGER_LOCK_MS holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): most_ms, yielded = deadlock_ms + 500, True @@ -572,7 +576,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: async def commit_enforced( conn: Any, confirm_lock: Callable[[], Any], - ready: Callable[[], Any], + ready: Callable[[], Awaitable[list[str]]], wait: Wait, log: Callable[[str], None], mark: Callable[[str], None], @@ -581,17 +585,17 @@ async def commit_enforced( session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before - each try (False skips the try while a process boots). After a failed try the in-flight + each try (the booting process ids skip the try). After a failed try the in-flight mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - if await ready(): - mark("unknown") + booting = await ready() + if not booting: try: - done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS), mark) except Refused: mark("recording") # the UPDATE's transaction rolled back raise @@ -601,6 +605,13 @@ async def commit_enforced( log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): + if booting: + raise Refused( + [ + Refusal(f"process {p}", "still booting at the end of --wait") + for p in booting + ] + ) raise Refused( [ Refusal( @@ -612,15 +623,18 @@ async def commit_enforced( ) -async def _mode_try(conn: Any, lock_ms: int) -> bool: - """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" +async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: + """One try including COMMIT, bounded by its lock and statement waits plus the standby + allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with conn.transaction(): + async with asyncio.timeout(bound_s), conn.transaction(): await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") await conn.execute( f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") status = await conn.execute( "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 65064ab365..f6535cdb40 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -469,8 +469,8 @@ async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None await _prepare(cell_stores, _Network()) logged: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async with pg.dedicated() as holder, pg.dedicated() as conn: tx = holder.transaction() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 429c2fcfb7..95fc006333 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json from collections.abc import Callable from dataclasses import dataclass, field @@ -244,8 +245,8 @@ def lock(sql: str, *_a: Any) -> str: async def nothing() -> None: pass - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait @@ -652,8 +653,8 @@ def lock(*_a: Any) -> str: async def confirm() -> None: confirmed.append(True) - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] logged: list[str] = [] await apg.commit_enforced( @@ -684,24 +685,108 @@ async def confirm() -> None: @pytest.mark.asyncio +@pytest.mark.parametrize("wait_s", [0, 2]) async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( clock: _Clock, + monkeypatch: pytest.MonkeyPatch, + wait_s: int, ) -> None: conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.own_fp = (CURRENT, NEXT) + + async def read(_ch: Any) -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name=None)] + + monkeypatch.setattr(beats, "read_beats", read) async def confirm() -> None: pass - async def booting() -> bool: - return False - - with pytest.raises(Refused, match="not taken within --wait"): + with pytest.raises(Refused) as error: await apg.commit_enforced( - conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + conn, + confirm, + run.ready_for_commit, + apg.Wait(wait_s, clock), + lambda _t: None, + lambda _m: None, ) + assert [str(r) for r in error.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: still booting at the end of --wait", + ] assert not any("LOCK TABLE" in s for s in conn.executed) +@pytest.mark.asyncio +@pytest.mark.parametrize("wait_s,bound_s", [(0, 8.1), (10, 10.0)]) +async def test_the_mode_try_bounds_a_commit_that_never_answers( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float +) -> None: + class _StalledCommit(FakeConn): + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + assert not exc[0] + conn.executed.append("COMMIT") + await asyncio.Future() + + return _Tx() + + conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) + bounds: list[float] = [] + timeout = asyncio.timeout + + def bounded(seconds: float) -> Any: + bounds.append(seconds) + return timeout(0) + + monkeypatch.setattr(apg.asyncio, "timeout", bounded) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(TimeoutError): + await asyncio.wait_for( + apg.commit_enforced( + conn, confirm, ready, apg.Wait(wait_s, clock), lambda _t: None, marks.append + ), + timeout=0.2, + ) + assert conn.executed[-1] == "COMMIT" + assert bounds == [bound_s] + assert marks == ["unknown"] + + +@pytest.mark.asyncio +async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: + conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(KeyboardInterrupt): + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append + ) + assert conn.executed[-1] == "ROLLBACK" + assert marks == [] + + @pytest.mark.asyncio async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( monkeypatch: pytest.MonkeyPatch, @@ -921,8 +1006,8 @@ async def test_wait_0_runs_one_try_of_each_wait( ) marks: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async def nothing() -> None: pass @@ -975,10 +1060,10 @@ def lock(*_a: Any) -> str: async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} - async def ready(_self: Any) -> bool: + async def ready(_self: Any) -> list[str]: if refused_before_any_try: raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) - return True + return [] async def confirm(_self: Any) -> None: pass From fefe1e04f092064190d35752f78adc42a42a0743 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:18:57 +0000 Subject: [PATCH 13/25] Require every cell probe grant before attach Check each probe privilege separately because a comma-separated privilege list accepts any one of them. Run the attach integration tests in the Postgres matrix and limit their heartbeat reads to the test's own process IDs while retaining the real queries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/cli/cell_attach_pg.py | 5 ++- tests/integration/test_cell_attach.py | 50 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 564393a0f7..22342c2cab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -898,6 +898,7 @@ jobs: tests/integration/test_router_directory_pg.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py + tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 60196363af..583d99775d 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -397,7 +397,10 @@ async def session_passes() -> bool: ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', - has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + has_table_privilege('loom.cell_probe', 'SELECT') + AND has_table_privilege('loom.cell_probe', 'INSERT') + AND has_table_privilege('loom.cell_probe', 'UPDATE') + AND has_table_privilege('loom.cell_probe', 'DELETE')), ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) ) v(need, held) WHERE NOT held diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index f6535cdb40..aaa48b8f0e 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -102,6 +102,21 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str ch = get_client() await ch.connect() ids: list[str] = [] + real_live, real_recent = instances.read_live, instances.read_recent + + async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, Any]]: + rows = await real_live(ch, all_deployments=all_deployments) + return [r for r in rows if r["instance_id"] in ids] + + async def read_recent( + ch: Any, *, window_s: float, since: str | None = None + ) -> list[dict[str, Any]]: + rows = await real_recent(ch, window_s=window_s, since=since) + return [r for r in rows if r["instance_id"] in ids] + + # The integration server shares ClickHouse and reports its own process heartbeats. + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) try: await pg_migrate.migrate() await pg.execute( @@ -669,6 +684,41 @@ async def pg_orgs(table: str, budget: Any) -> list[str]: assert grade == "WARN" and detail.endswith(": loom.it_large") +@pytest.mark.asyncio +async def test_attach_requires_every_probe_privilege(cell_stores) -> None: + role = f"it_probe_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + privileges = ("SELECT", "INSERT", "UPDATE", "DELETE") + grants = [ + ("SELECT",), + *(tuple(p for p in privileges if p != missing) for missing in privileges), + privileges, + ] + observed: dict[tuple[str, ...], list[str]] = {} + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + for held in grants: + await conn.execute(f"REVOKE ALL ON loom.cell_probe FROM {role}") + await conn.execute(f"GRANT {', '.join(held)} ON loom.cell_probe TO {role}") + await conn.execute(f"SET ROLE {role}") + refusals = await cell_attach_pg.own_role_refusals(conn, []) + await conn.execute("RESET ROLE") + observed[held] = [r.reason for r in refusals if "loom.cell_probe" in r.reason] + assert observed == { + held: ( + [] + if held == privileges + else ["attach's role lacks SELECT, INSERT, UPDATE and DELETE on loom.cell_probe"] + ) + for held in grants + } + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( cell_stores, From bd4cda33fd5d2bbbbc26812b1be8b3bec9c0212a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:29:02 +0000 Subject: [PATCH 14/25] Keep cell attach deadlines from waiting on rollback Manage mode and direct-session transactions explicitly so a canceled statement cannot start an unbounded rollback. Roll back ordinary mode statement errors within the try's deadline; terminate synchronously when a try ends with an error or cancellation. Signed-off-by: zlareb1 --- server/cli/cell_attach_pg.py | 64 +++++++++++------- tests/unit/test_cell_attach.py | 116 ++++++++++++++++++++++++++++----- 2 files changed, 139 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 583d99775d..d58b553d7a 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -122,10 +122,18 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: startup parameters drops it). Behind a transaction pooler a session-level lock would outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" - async with conn.transaction(): - first = await conn.fetchrow(_IDENTITY) - async with conn.transaction(): - second = await conn.fetchrow(_IDENTITY) + identities = [] + try: + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() + except BaseException: + # A cancelled read must not wait for a transaction context's rollback. + conn.terminate() + raise + first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): out.append( @@ -627,31 +635,39 @@ async def commit_enforced( async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: - """One try including COMMIT, bounded by its lock and statement waits plus the standby - allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + """One try including rollback or COMMIT, bounded by its lock and statement waits plus + the standby allowance; False after a statement timeout rolls back, TimeoutError at + the try's deadline.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with asyncio.timeout(bound_s), conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + async with asyncio.timeout(bound_s): + tx = conn.transaction() + await tx.start() + try: + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + except Exception as e: + await tx.rollback() + if getattr(e, "sqlstate", None) in _TIMED_OUT: + return False + raise + await tx.commit() return True - except Refused: + except BaseException: + # Termination is synchronous: cancellation never starts an unbounded rollback. + conn.terminate() raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise - return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 95fc006333..065f3a455d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -473,14 +473,29 @@ def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await conn.execute("BEGIN") + + async def commit(self) -> None: + await conn.execute("COMMIT") + + async def rollback(self) -> None: + await conn.execute("ROLLBACK") + async def __aenter__(self) -> None: - conn.executed.append("BEGIN") + await self.start() async def __aexit__(self, *exc: Any) -> None: - conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + if exc[0]: + await self.rollback() + else: + await self.commit() return _Tx() + def terminate(self) -> None: + self.executed.append("TERMINATE") + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -663,6 +678,8 @@ async def ready() -> list[str]: assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 + assert conn.executed.count("ROLLBACK") == 1 + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio @@ -725,19 +742,11 @@ async def test_the_mode_try_bounds_a_commit_that_never_answers( monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float ) -> None: class _StalledCommit(FakeConn): - def transaction(self) -> Any: - conn = self - - class _Tx: - async def __aenter__(self) -> None: - conn.executed.append("BEGIN") - - async def __aexit__(self, *exc: Any) -> None: - assert not exc[0] - conn.executed.append("COMMIT") - await asyncio.Future() - - return _Tx() + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "COMMIT": + await asyncio.Future() + return result conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] @@ -763,11 +772,84 @@ async def ready() -> list[str]: ), timeout=0.2, ) - assert conn.executed[-1] == "COMMIT" + assert "COMMIT" in conn.executed + assert conn.executed[-1] == "TERMINATE" assert bounds == [bound_s] assert marks == ["unknown"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["LOCK TABLE", "UPDATE", "ROLLBACK"]) +async def test_a_mode_deadline_never_waits_for_rollback( + monkeypatch: pytest.MonkeyPatch, stall: str +) -> None: + class _StalledRollback(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "ROLLBACK" and sql.startswith("LOCK TABLE"): + raise _Timeout() + if sql.startswith(stall) or sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + marks: list[str] = [] + task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "mode deadline is stuck awaiting rollback" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + assert marks == (["unknown"] if stall == "UPDATE" else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_read", [1, 2]) +@pytest.mark.parametrize("failure", ["cancel", "error"]) +async def test_a_failed_direct_session_read_never_waits_for_rollback( + failed_read: int, failure: str +) -> None: + class _StalledRollback(FakeConn): + reads = 0 + + async def fetchrow(self, sql: str, *args: Any) -> Any: + self.reads += 1 + if self.reads == failed_read: + if failure == "error": + raise ValueError("identity read failed") + await asyncio.Future() + return (1, "t0") + + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + task = asyncio.create_task(apg.direct_refusals(conn, "attach")) + await asyncio.sleep(0) + if failure == "cancel": + task.cancel() + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "direct-session failure is stuck awaiting rollback" + if failure == "cancel": + assert task.cancelled() + else: + assert isinstance(task.exception(), ValueError) + assert conn.executed[-1] == "TERMINATE" + assert "ROLLBACK" not in conn.executed + + @pytest.mark.asyncio async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) @@ -783,7 +865,7 @@ async def ready() -> list[str]: await apg.commit_enforced( conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append ) - assert conn.executed[-1] == "ROLLBACK" + assert conn.executed[-1] == "TERMINATE" assert marks == [] From 3ba813ccd9b7c40f355e6bba537e5d7f1b6231c6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:44:09 +0000 Subject: [PATCH 15/25] Bound attach trigger attempts and scope gate cleanup to its run Give each trigger attempt a client deadline covering migration lock acquisition, DDL, commit, rollback and unlock. Use explicit transactions so cancellation terminates the session without waiting for rollback or masking the original error with lock cleanup. Give gate batch scopes a unique prefix per run so cleanup preserves existing rows. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 8 ++- server/cli/cell_attach_pg.py | 47 ++++++++------ server/core/cell_triggers.py | 21 +++++- tests/unit/test_bench_cell_trigger_gate.py | 53 +++++++++++++++ tests/unit/test_cell_attach.py | 75 ++++++++++++++++++++++ tests/unit/test_cell_org_triggers.py | 14 ++-- 6 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_bench_cell_trigger_gate.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 4363a810d6..8797b4b8e7 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -80,8 +80,9 @@ async def _batch(args: argparse.Namespace, marked: bool) -> float: t = time.perf_counter() await conn.execute( "INSERT INTO loom.scope_generations (org, scope) " - "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + "SELECT $1, $2 || g || '-' || $3 FROM generate_series(1, 1000) g", args.org, + args.scope_prefix, uuid.uuid4().hex[:8], ) times.append((time.perf_counter() - t) * 1000) @@ -98,6 +99,7 @@ async def main() -> None: p.add_argument("--org", required=True) p.add_argument("--namespace", required=True) args = p.parse_args() + args.scope_prefix = f"gate-{uuid.uuid4().hex}-" try: for _ in range(ROUNDS): for on in (True, False): @@ -120,7 +122,9 @@ async def _clean(args: argparse.Namespace) -> None: conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) try: await conn.execute( - "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + "DELETE FROM loom.scope_generations WHERE org = $1 AND starts_with(scope, $2)", + args.org, + args.scope_prefix, ) finally: await conn.close() diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d58b553d7a..2af085f86f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -31,6 +31,7 @@ _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound # A trigger try yields quickly to active writers. _TRIGGER_LOCK_MS = 100 # not a setting: one try's bound +_TRIGGER_TRY_MARGIN_S = 8.0 # not a setting: catalog, DDL and standby allowance # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait @@ -524,29 +525,37 @@ async def _migration_holder(conn: Any, key: int) -> str: async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the - try floor. A busy migration lock answers `busy` naming that lock's holder.""" + try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or + rollback. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + bound_s = ( + wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) + ) / 1000 + _TRIGGER_TRY_MARGIN_S try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) == "55P03": - # A boot migrating: the next try. - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + async with asyncio.timeout(bound_s): + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + try: + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) + return outcome + except BaseException: + # Closing the session releases its locks without asynchronous error cleanup. + conn.terminate() raise - finally: - await conn.execute("RESET lock_timeout") - try: - return await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - finally: - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 7a1ee5e5b7..1899a46bab 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -328,18 +328,32 @@ async def _run_ddl( import asyncpg for attempt in range(attempts): + rolled_back = False try: - async with conn.transaction(): + tx = conn.transaction() + await tx.start() + try: await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) + except Exception: + await tx.rollback() + rolled_back = True + raise + await tx.commit() return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: + if not rolled_back: + conn.terminate() + raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: + if not rolled_back: + conn.terminate() + raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) @@ -354,6 +368,11 @@ async def _run_ddl( held, ) return Outcome("busy", held) + except BaseException: + if not rolled_back: + # Cancellation must not wait for another statement on this session. + conn.terminate() + raise return Outcome("busy") diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py new file mode 100644 index 0000000000..89f32cddde --- /dev/null +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -0,0 +1,53 @@ +"""The trigger gate deletes only the scope rows its own batch inserts wrote.""" + +from __future__ import annotations + +import argparse +import sqlite3 +from typing import Any + +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.mark.asyncio +async def test_batch_cleanup_preserves_preexisting_gate_scopes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = sqlite3.connect(":memory:") + db.execute("ATTACH DATABASE ':memory:' AS loom") + db.execute("CREATE TABLE loom.scope_generations (org text, scope text)") + db.create_function("starts_with", 2, lambda value, prefix: value.startswith(prefix)) + prefix = "gate-own-run-" + sentinel = [("org", "gate-existing"), ("other", prefix + "existing")] + db.executemany("INSERT INTO loom.scope_generations VALUES (?, ?)", sentinel) + + class _Conn: + async def execute(self, sql: str, *args: Any) -> None: + sql = sql.replace( + "FROM generate_series(1, 1000) g", + "FROM (WITH RECURSIVE seq(g) AS (VALUES(1) UNION ALL " + "SELECT g + 1 FROM seq WHERE g < 1000) SELECT g FROM seq)", + ) + db.execute(sql, {str(i): value for i, value in enumerate(args, 1)}) + + async def close(self) -> None: + pass + + async def connect(*_a: Any, **_kw: Any) -> Any: + return _Conn() + + monkeypatch.setattr(gate.asyncpg, "connect", connect) + monkeypatch.setattr(gate, "INSERTS", 1) + args = argparse.Namespace(pg_dsn="unused", org="org", scope_prefix=prefix) + try: + await gate._batch(args, True) + assert db.execute("SELECT count(*) FROM loom.scope_generations").fetchone()[0] == 1002 + await gate._clean(args) + assert ( + db.execute("SELECT org, scope FROM loom.scope_generations ORDER BY org").fetchall() + == sentinel + ) + finally: + db.close() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 065f3a455d..6d57cbb740 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -1038,6 +1038,81 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: assert asked == [f"{apg.TRY_FLOOR_MS}ms"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) +async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str +) -> None: + import asyncpg + + from server.core import cell_triggers + + class _Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "rollback" and sql == "CREATE TRIGGER stub": + raise asyncpg.LockNotAvailableError("busy") + needles = { + "migration": "SELECT pg_advisory_lock", + "reset": "RESET lock_timeout", + "ddl": "CREATE TRIGGER stub", + "commit": "COMMIT", + "rollback": "ROLLBACK", + "unlock": "SELECT pg_advisory_unlock", + } + if sql.startswith(needles[stall]) or (stall == "ddl" and sql == "ROLLBACK"): + await asyncio.Future() + return result + + async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: + return await cell_triggers._run_ddl( + conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False + ) + + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = _Stalled() + task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) + done, _ = await asyncio.wait([task], timeout=0.2) + try: + assert task in done, f"trigger deadline is stuck at {stall}" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + if stall in {"migration", "ddl", "commit", "rollback"}: + assert apg._MIGRATION_UNLOCK not in conn.executed + finally: + for _ in range(3): + if task.done(): + break + task.cancel() + await asyncio.wait([task], timeout=0.01) + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["migration", "ddl"]) +async def test_a_failed_trigger_try_does_not_mask_its_error_with_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, failed: str +) -> None: + from server.core import cell_triggers + + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + raise ValueError("DDL failed") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn( + [ + ("pg_advisory_lock", ValueError("migration failed") if failed == "migration" else "OK"), + ("RESET", RuntimeError("reset failed") if failed == "migration" else "OK"), + ("pg_advisory_unlock", RuntimeError("unlock failed")), + ] + ) + with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): + await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) + assert conn.executed[-1] == "TERMINATE" + + @pytest.mark.asyncio async def test_a_busy_migration_lock_is_named_as_such( monkeypatch: pytest.MonkeyPatch, clock: _Clock diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index decb365521..769cfe1bb3 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -617,11 +617,17 @@ def __init__(self, error: Exception) -> None: self.error = error def transaction(self) -> Any: - @contextlib.asynccontextmanager - async def _tx() -> AsyncIterator[None]: - yield + class _Tx: + async def start(self) -> None: + pass - return _tx() + async def commit(self) -> None: + pass + + async def rollback(self) -> None: + pass + + return _Tx() async def fetchrow(self, _sql: str, *_a: Any) -> dict[str, Any]: return { From 98fc3331237b6c9f5828e899b2abe1b7d6c2fdbb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:30:52 +0000 Subject: [PATCH 16/25] Operator placement durably records its directory winner and local tenancy Signed-off-by: zlareb1 --- docs/architecture/cells.md | 23 +- docs/reference/cli.md | 1 + server/cli/cell.py | 2 + server/cli/cell_place.py | 166 +++++++ server/core/cell.py | 7 +- server/router/app.py | 46 +- server/router/directory.py | 60 ++- tests/integration/test_cell_place.py | 238 +++++++++ tests/integration/test_router_directory_pg.py | 28 +- tests/unit/snapshots/cli_help.json | 8 +- tests/unit/snapshots/cli_tree.json | 99 ++++ tests/unit/test_cell_place.py | 466 ++++++++++++++++++ tests/unit/test_router.py | 4 +- tests/unit/test_router_attach.py | 15 +- 14 files changed, 1108 insertions(+), 55 deletions(-) create mode 100644 server/cli/cell_place.py create mode 100644 tests/integration/test_cell_place.py create mode 100644 tests/unit/test_cell_place.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index f32815b635..4c5fec9a85 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -754,13 +754,21 @@ read-only scope, lists any tenancy row the directory does not place. it, unless the cell is the first and its mode reads `recording`, and an org-free command never uses it; a leaked one is stopped by `loom cell limit --closed`, which needs none of the cell's stores, and replaced by `loom cell credential` (below). - `loom cell place --org ` (declared org-free: it writes the org's placement and - tenancy row, which is how an org comes to be held), with the router's operator credential, + `loom cell place --cell --org ` (declared org-free: it writes the org's + placement and tenancy row, which is how an org comes to be held), with the router's operator credential, places an org for a command that writes a new one (a bot, an import), through the router's placement API and in its order, as attach's placement of configured orgs does: it inserts the placement and reads back the winner, writes the cell's tenancy row only if the winner is this cell (otherwise it refuses and writes nothing), and reports success only once the - row is written, so a rerun is idempotent. For an org the directory does not place yet it + row commits synchronously. The CLI holds the attach lock on a direct marked Postgres + session, requires a full enforced local cell state and the full accepted configuration + fingerprint, and checks both again before inserting. A transactional durability message + makes even a no-op insert flush an earlier primary-only tenancy row to the standby. + Opening the session, local statements, COMMIT and cleanup have client deadlines derived + from `LOOM_PG_STATEMENT_TIMEOUT_MS`. An unknown outcome names rerunning the same command; + the permanent directory winner and idempotent local insert let that rerun finish. The + route checks directory state; a direct operator API caller is trusted to check local + Postgres mode as the CLI does. For an org the directory does not place yet it passes the placement API's gate (below): refused while the directory is unavailable, for a cell the gate does not accept, closed, or when the fingerprint of the CLI's own configuration of the cell (with the deployment id read from the stores it connects to) @@ -859,9 +867,12 @@ read-only scope, lists any tenancy row the directory does not place. safe, since the read-back makes it idempotent. For an org the directory already places, a cell's call is checked only for its credential (against the cells' hashes the router caches for `LOOM_ROUTER_DIRECTORY_CACHE_S`, default 5 s) and the cell's - rate, and a `loom cell place` only for the operator credential; both answer the winner - whatever the cell's state, quota or fingerprint, adding no placement (the winner is - public through the lookup anyway); a request refused past its cell's rate answers 503 + rate. An operator's `loom cell place` checks its deployment id and fingerprint against + the requested cell's locked row even for an org already placed there; an org placed + elsewhere answers that winner and the CLI refuses without inserting local tenancy. + Existing placements bypass closed and quota checks. Every successful call writes an + audit row before its synchronous commit, including a retry of an existing winner, so + the returned winner survives failover. A request refused past its cell's rate answers 503 with `Retry-After` (at most a minute), and its retry is admitted by the replica that answers it once the cell's calls there are under the rate. An org the cell already holds needs no call, so serving it never depends on the router. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 12bac9a65b..dd75540091 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -329,6 +329,7 @@ python scripts/codegen/docs_reference.py | `loom cell limit` | Close, reopen or bound a cell to new orgs. | `--cell`, `--closed`, `--open`, `--placements-max`, `--router` | | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | +| `loom cell place` | Place an org in this cell and record its tenancy. | `--cell`, `--org`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | | `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | diff --git a/server/cli/cell.py b/server/cli/cell.py index 65f18990d8..c14927994c 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -460,7 +460,9 @@ def _add_cell_parser(sub: Any) -> None: remove.set_defaults(func=_cmd_remove) from server.cli.cell_attach import add_attach_parser + from server.cli.cell_place import add_place_parser from server.cli.cell_prepare import add_prepare_parser + add_place_parser(csub) add_prepare_parser(csub) add_attach_parser(csub) diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py new file mode 100644 index 0000000000..3820ecf2cc --- /dev/null +++ b/server/cli/cell_place.py @@ -0,0 +1,166 @@ +"""Directory-first operator placement, followed by a durable local tenancy row.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_pg as apg +from server.cli._ch import run_connected, writes +from server.cli.cell_ch import Refused +from server.core import cell, pg +from server.router.directory import PLACE_MAX_S +from server.settings import settings + + +class PlacementRefused(Exception): + """A placement cannot complete with this configuration or directory winner.""" + + +def _bound_s() -> float: + return settings.pg_statement_timeout_ms / 1000 + 1 + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return "--cell names 1 to 32 of a-z, 0-9, -" + if args.cell != settings.cell: + return f"place runs with the cell's own configuration: set LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return "place needs LOOM_STORAGE_AUTHORITY=postgres and the cell's LOOM_PG_DSN" + if cell.named_org(args.org) is None: + return "--org names a nonempty Postgres org" + return cell.org_name_refusal(args.org) + + +async def _identity(conn: Any, name: str) -> tuple[str, dict[str, Any]]: + """Read the full local cell state and this CLI's accepted configuration.""" + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + state, first = await cell.read_state_if_present(conn) + if state != "enforced" or first != name: + raise PlacementRefused( + f"local cell state is {state!r}, naming {first!r}; place needs enforced {name!r}" + ) + deployment = await conn.fetchval("SELECT deployment_id FROM loom.authority") + if not isinstance(deployment, str) or not deployment: + raise PlacementRefused("the local Postgres records no deployment id") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + mine = cell.fingerprint(deployment) + if own is None or not any( + cell.matches(mine, cell.json_object(own[k])) for k in ("current_fp", "next_fp") + ): + raise PlacementRefused("the CLI's configuration matches no local cell fingerprint") + return deployment, cell.as_value(mine) + + +async def _directory( + router: httpx.URL, name: str, org: str, deployment: str, fingerprint: dict[str, Any] +) -> None: + url = cell_cli.at(router, f"/cells/v1/operator/cells/{name}/place") + async with httpx.AsyncClient(trust_env=router.scheme == "https") as client: + answer = await client.post( + url, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, + timeout=httpx.Timeout(PLACE_MAX_S + 10.0, connect=5.0), + ) + posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) + if posted is None: + raise PlacementRefused(cell_cli.not_the_router(answer)) + if answer.status_code != 200: + raise PlacementRefused(f"the router refused ({answer.status_code}): {posted['error']}") + if posted.get("cell") != name: + raise PlacementRefused( + f"org {org!r} belongs to cell {posted.get('cell')!r}; no local tenancy row written" + ) + if posted.get("deployment_id") != deployment: + raise PlacementRefused("the router answered another deployment id") + + +async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: + """Hold the operator lock through directory placement and an acknowledged local commit. + Every local phase, including opening and closing the direct session, has a deadline.""" + cm = pg.dedicated(statement_timeout_ms=settings.pg_statement_timeout_ms) + conn = None + try: + async with asyncio.timeout(_bound_s()): + conn = await cm.__aenter__() + async with asyncio.timeout(_bound_s()): + if refusals := await apg.direct_refusals(conn, "place"): + raise Refused(refusals) + await apg.take_lock(conn, "place") + deployment, fingerprint = await _identity(conn, args.cell) + await _directory(router, args.cell, args.org, deployment, fingerprint) + async with asyncio.timeout(_bound_s()): + tx = conn.transaction() + await tx.start() + if not await apg.lock_held(conn): + raise PlacementRefused("the operator lock was lost") + if await _identity(conn, args.cell) != (deployment, fingerprint): + raise PlacementRefused( + "the local identity or configuration changed during placement" + ) + await conn.execute( + "INSERT INTO loom.tenancy (org) VALUES ($1) ON CONFLICT DO NOTHING", args.org + ) + # A no-op insert must also flush a primary-only predecessor to the standby. + await conn.execute(pg._DURABLE_MESSAGE) + await tx.commit() + finally: + if conn is not None: + try: + async with asyncio.timeout(_bound_s()): + await cm.__aexit__(None, None, None) + finally: + conn.terminate() + return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + +def cmd_place(args: argparse.Namespace) -> int: + if reason := _args_problem(args): + print(f"loom cell place: {reason}", file=sys.stderr) + return 2 + router = cell_cli.operator_router(args) + if isinstance(router, str): + print(f"loom cell place: {router}", file=sys.stderr) + return 2 + + async def run(ch: Any) -> int: + try: + result = await place(args, router) + except asyncio.CancelledError: + print( + "loom cell place: stopped (CancelledError); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + raise + except Exception as e: # noqa: BLE001 — an unknown outcome must name its idempotent retry + print( + f"loom cell place: stopped ({type(e).__name__}: {e}); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + return run_connected( + run, org_free=writes("places an org in the directory and records its local tenancy") + ) + + +def add_place_parser(csub: Any) -> None: + p = csub.add_parser("place", help="Place an org in this cell and record its tenancy.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--org", required=True, help="The org to place") + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_place) diff --git a/server/core/cell.py b/server/core/cell.py index 37e20f99cb..fa1a44615e 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -415,10 +415,9 @@ def _unset_reason(view: _View) -> str: ) -# The fingerprint fields a CLI's gate judges: the stores it would write. A CLI verifies -# no token and its fingerprint never reaches the router, so the rest of the servers' -# fingerprint is not its to match (docs/architecture/cells.md, "The CLI declares its -# orgs"). +# An org-declaring CLI's gate checks the stores it would write: it verifies no token +# and sends no fingerprint to the router. Operator placement checks the full fingerprint +# separately (docs/architecture/cells.md, "The CLI declares its orgs"). _STORE_FIELDS = ("deployment_id", "bucket", "bucket_prefix") diff --git a/server/router/app.py b/server/router/app.py index 4e24a7c922..7c80df19af 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -14,6 +14,8 @@ - `POST /cells/v1/operator/cells//limit` and `.../credential`: the operator's changes to a cell's row (`loom cell limit`, `loom cell credential`), with the router's operator credential; +- `POST /cells/v1/operator/cells//place`: `loom cell place` places an org with the + operator credential, verifying the requested cell's deployment id and fingerprint; - `POST /cells/v1/operator/cells//register`: `loom cell prepare` registers a cell, once the probe listener (the first cell's at LOOM_ROUTER_UPSTREAM_PROBE_URL, a later cell's at its `probe_url`) answers its deployment id, prepare's nonces and the bucket and @@ -28,9 +30,9 @@ Every refusal of a placement writes nothing. A malformed placement body answers 400; the directory's refusals answer 503, so the cell's request answers 503 and its retry is safe, except a new org longer than `cell.ORG_MAX_BYTES`, which answers 400 `ORG_TOO_LONG` and no -retry places; a credential that names no cell, or not this one, answers 401. An operator -route answers a cell the directory does not hold 404 with the code `NO_SUCH_CELL`, so a 404 -without it is a router of an earlier build. +retry places; a credential that names no cell, or not this one, answers 401. Operator +placement refusals answer 409; the other operator routes answer a cell the directory does +not hold 404 with `NO_SUCH_CELL`, so a 404 without it is a router of an earlier build. """ from __future__ import annotations @@ -301,7 +303,7 @@ def refusal(locked: directory.Cell) -> str: return "" try: - winner, deployment_id = await directory.place(org, name, refusal) + winner, deployment_id = await directory.place(org, name, refusal, actor=f"cell:{name}") except directory.OrgTooLong as e: return _refused(400, e.reason, code="ORG_TOO_LONG") except (directory.PlacementRefused, directory.DirectoryUnavailable) as e: @@ -314,6 +316,41 @@ def refusal(locked: directory.Cell) -> str: log.info("placement of %s: %s (asked by %s)", org, winner, name) return JSONResponse({"cell": winner, "deployment_id": deployment_id}) + async def operator_place(self, request: Request) -> JSONResponse: + """Place with the operator credential. The CLI checks its local Postgres mode; + this route trusts that check and verifies the directory's locked identity.""" + body = await self._operator_body(request) + if isinstance(body, JSONResponse): + return body + org = cell_state.named_org(body.get("org")) + deployment_id = _text(body, "deployment_id", needed=True) + if ( + set(body) != {"org", "deployment_id", "fingerprint"} + or org is None + or deployment_id is None + ): + return _refused(400, "a placement names org, deployment_id and fingerprint") + if reason := cell_state.org_name_refusal(org): + return _refused(400, reason, code="ORG_TOO_LONG") + name = request.path_params["cell"] + + def verify(locked: directory.Cell) -> str: + if locked.deployment_id != deployment_id: + return "the calling process names another deployment id" + if not fingerprint_matches(locked, body["fingerprint"]): + return f"the calling process's configuration matches no value of {name!r}" + return "" + + try: + winner, deployment = await directory.place( + org, name, placement_refusal, actor="operator", verify=verify + ) + except directory.PlacementRefused as e: + return _refused(409, e.reason) + except directory.DirectoryUnavailable as e: + return _refused(503, e.reason, code="BACKEND_UNAVAILABLE") + return JSONResponse({"org": org, "cell": winner, "deployment_id": deployment}) + async def cell_state(self, request: Request) -> JSONResponse: """`GET /cells/v1/cells/`: the first cell's state, for its own credential only, from this replica's last read of the row (`_state_row`). A first cell in @@ -1365,6 +1402,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: return Starlette( routes=[ Route("/cells/v1/placements", router.place, methods=["POST"]), + Route("/cells/v1/operator/cells/{cell}/place", router.operator_place, methods=["POST"]), Route("/cells/v1/cells/{cell}", router.cell_state, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/limit", router.set_limit, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}/register", router.register, methods=["POST"]), diff --git a/server/router/directory.py b/server/router/directory.py index a92967ed86..52893ef4ef 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -12,7 +12,8 @@ placing one org at once agree on the winner, and a retry of the same call answers the same; a batch takes none, since no other cell exists while it runs; - the placements' primary key, as a backstop: every insert is `ON CONFLICT DO NOTHING`, and - only a row it returns is counted and audited. + only a row it returns is counted. Every successful call writes an audit row before + committing, so a duplicate winner waits for synchronous durability too. """ from __future__ import annotations @@ -59,6 +60,10 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) +# New placement: BEGIN, org lock, winner, cell lock, winner, insert, counter, winner, +# audit, COMMIT. The audit also gives duplicate winners a synchronous durability fence. +PLACE_STATEMENTS = 10 +PLACE_MAX_S = _bound_s(PLACE_STATEMENTS) # not a setting: derived # The longest a limit or credential change can take: eight statements (begin, lock, row, # check, update, audit, read-back, commit). Registration has its own bound, below. OPERATOR_CHANGE_MAX_S = _bound_s(8) # not a setting: derived from the bound above @@ -635,39 +640,49 @@ async def first_cell_named(cell_name: str) -> Cell | None: return _cell(row) if row else None -async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: - """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) - that holds it. An org placed already answers its cell, whatever the cell's state. - Otherwise the cell's row is locked, the winner read again (an attach batch holds the row - lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it - waits for the placement and the quota holds across router replicas; a refused placement - raises PlacementRefused and writes nothing, and a new org name longer than - `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as - any placed org does).""" +async def place( + org: str, + cell_name: str, + refusal: Callable[[Cell], str], + *, + actor: str, + verify: Callable[[Cell], str] | None = None, +) -> tuple[str, str]: + """The permanent winner of `org`, with every successful call audited and committed + synchronously. `verify` judges the requested cell's locked identity even for a winner + already there; `refusal` and quota judge new placements only. A winner elsewhere needs + neither check. The org lock and cell row lock serialize live and attach placements.""" async with _reading(), _connection() as conn, conn.transaction(): - # Placements of one org run one at a time, whichever cell asks: the later one - # reads the earlier one's winner. await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) row = await conn.fetchrow(_WINNER, org) - if row is None: - _refuse_long([org]) + if row is None or (verify is not None and row["cell"] == cell_name): + if row is None: + _refuse_long([org]) locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) if locked is None: raise PlacementRefused(f"cell {cell_name!r} is not in the directory") - # An attach batch of this cell, which holds the row lock and not the org's, may - # have placed the org since the first read: it is answered as placed. + candidate = _cell(locked) + # An attach batch holds this row's lock rather than the org's. row = await conn.fetchrow(_WINNER, org) + if (row is None or row["cell"] == cell_name) and verify is not None: + if reason := verify(candidate): + raise PlacementRefused(reason) if row is None: - await _place_one(conn, org, cell_name, _cell(locked), refusal) + await _place_one(conn, org, cell_name, candidate, refusal) row = await conn.fetchrow(_WINNER, org) - return str(row["cell"]), str(row["deployment_id"]) + winner = str(row["cell"]), str(row["deployment_id"]) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + actor, + {"org": org, "cell": winner[0]}, + ) + return winner async def _place_one( conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] ) -> None: - """Judge the locked row (`refusal`, then its quota) and insert the placement; its count - and audit row only when the insert wrote it.""" + """Judge the locked row and insert the placement, counted only when written.""" if reason := refusal(candidate): raise PlacementRefused(reason) quota = candidate.placements_max @@ -684,11 +699,6 @@ async def _place_one( await conn.execute( "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name ) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", - f"cell:{cell_name}", - {"org": org, "cell": cell_name}, - ) def _refuse_long(orgs: Iterable[str]) -> None: diff --git a/tests/integration/test_cell_place.py b/tests/integration/test_cell_place.py new file mode 100644 index 0000000000..7024024002 --- /dev/null +++ b/tests/integration/test_cell_place.py @@ -0,0 +1,238 @@ +"""Operator placement on two cells' databases and a real directory, without memory rows.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import os +import time + +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.router import app, directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database, own_database + +OPERATOR = "lco_" + "3" * 128 +ROUTER = httpx.URL("http://127.0.0.1:7901") + + +@pytest.fixture +async def stores(monkeypatch): + admin = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, dsn = await create_database(admin) + async with contextlib.AsyncExitStack() as stack: + stack.push_async_callback(drop_database, admin, name) + first = await stack.enter_async_context(own_database()) + second = await stack.enter_async_context(own_database()) + monkeypatch.setattr(directory, "_pool", None) + stack.push_async_callback(directory.close) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + await directory.migrate() + pg.expect_deployment(None) + router_app = app.build_app(app.Router()) + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kw: real_client(transport=httpx.ASGITransport(app=router_app), **kw), + ) + targets = {"first": first, "second": second} + try: + for cell_name, cell_dsn in targets.items(): + await _initialize(cell_dsn, cell_name) + yield targets + finally: + pg.expect_deployment(None) + await pg.close_pool() + await directory.close() + + +async def _initialize(dsn, name): + import asyncpg + + fields = cell.fingerprint(f"dep-{name}") + conn = await asyncpg.connect(dsn) + try: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + f"dep-{name}", + ) + await conn.execute( + "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", name == "first", name + ) + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6)", + name, + name == "first", + f"{name}.example.test", + hashlib.sha256(name.encode()).hexdigest(), + f"dep-{name}", + cell.as_value(fields), + ) + + +async def _local(stores, monkeypatch, name, org="acme"): + monkeypatch.setattr(settings, "cell", name) + monkeypatch.setattr(settings, "pg_dsn", stores[name]) + return await cell_place.place(argparse.Namespace(cell=name, org=org), ROUTER) + + +async def _tenancy(dsn): + import asyncpg + + conn = await asyncpg.connect(dsn) + try: + return [r["org"] for r in await conn.fetch("SELECT org FROM loom.tenancy ORDER BY org")] + finally: + await conn.close() + + +async def test_operator_place_retries_and_refuses_a_different_cell( + stores, monkeypatch, record_property +): + started = time.perf_counter() + first = await _local(stores, monkeypatch, "first") + record_property("local_first_place_ms", (time.perf_counter() - started) * 1000) + assert first == {"org": "acme", "cell": "first", "deployment_id": "dep-first"} + started = time.perf_counter() + assert await _local(stores, monkeypatch, "first") == first + record_property("local_duplicate_place_ms", (time.perf_counter() - started) * 1000) + assert await _tenancy(stores["first"]) == ["acme"] + with pytest.raises(cell_place.PlacementRefused, match="belongs to cell 'first'"): + await _local(stores, monkeypatch, "second") + assert await _tenancy(stores["second"]) == [] + async with directory._connection() as conn: + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'first'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 3 + ) + + +async def test_two_cells_racing_one_org_have_one_winner_and_count_once(stores): + async def place(name): + fields = cell.as_value(cell.fingerprint(f"dep-{name}")) + async with httpx.AsyncClient() as client: + return await client.post( + str(ROUTER.copy_with(path=f"/cells/v1/operator/cells/{name}/place")), + json={"org": "raced", "deployment_id": f"dep-{name}", "fingerprint": fields}, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + a, b = await asyncio.gather(place("first"), place("second")) + assert a.status_code == b.status_code == 200 + assert a.json() == b.json() + async with directory._connection() as conn: + assert await conn.fetchval("SELECT sum(placed) FROM loom_router.cells") == 1 + assert await conn.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + assert await _tenancy(stores["first"]) == await _tenancy(stores["second"]) == [] + + +@pytest.mark.parametrize("limit", ["closed = true", "placements_max = 0"]) +async def test_new_closed_or_full_cell_writes_neither_store(stores, monkeypatch, limit): + async with directory._connection() as conn: + await conn.execute(f"UPDATE loom_router.cells SET {limit} WHERE cell = 'first'") + with pytest.raises(cell_place.PlacementRefused, match="router refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert await directory.lookup("acme") is None + + +async def test_failed_local_insert_resumes_and_existing_winner_ignores_closed_quota( + stores, monkeypatch +): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute( + "CREATE FUNCTION loom.fail_place() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'local insertion refused'; END $$" + ) + await conn.execute( + "CREATE TRIGGER fail_place BEFORE INSERT ON loom.tenancy FOR EACH ROW EXECUTE FUNCTION loom.fail_place()" + ) + with pytest.raises(asyncpg.RaiseError, match="local insertion refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert (await directory.lookup("acme"))[0] == "first" + await conn.execute("DROP TRIGGER fail_place ON loom.tenancy") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +async def test_first_recording_refuses_and_enforced_attaching_succeeds(stores, monkeypatch): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + with pytest.raises(cell_place.PlacementRefused, match="needs enforced"): + await _local(stores, monkeypatch, "first") + assert await directory.lookup("acme") is None + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET state = 'attaching' WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +@pytest.mark.parametrize("changed", ["mode", "fingerprint"]) +async def test_changed_local_state_after_directory_call_refuses_insert( + stores, monkeypatch, changed +): + import asyncpg + + real = cell_place._directory + + async def change_after_call(*args): + await real(*args) + conn = await asyncpg.connect(stores["first"]) + try: + if changed == "mode": + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + else: + fields = cell.fingerprint("dep-first") + fields["oidc_client_id"] = "changed" + await conn.execute( + "UPDATE loom.cell_self SET current_fp = $1::jsonb", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + + monkeypatch.setattr(cell_place, "_directory", change_after_call) + with pytest.raises(cell_place.PlacementRefused): + await _local(stores, monkeypatch, "first") + assert (await directory.lookup("acme"))[0] == "first" + assert await _tenancy(stores["first"]) == [] diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index caf092e9ec..94b25a95a9 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -109,7 +109,8 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, org ) racing = asyncio.gather( - directory.place(org, a, _never), directory.place(org, b, _never) + directory.place(org, a, _never, actor="operator"), + directory.place(org, b, _never, actor="operator"), ) # Both placements wait on the org's lock before it is released. for _ in range(100): @@ -133,10 +134,11 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT actor FROM loom_router.audit WHERE action = 'place' AND detail->>'org' = $1", org, ) - assert [r["actor"] for r in audit] == [f"cell:{winners[0][0]}"] + assert [r["actor"] for r in audit] == ["operator", "operator"] # Ten concurrent placements in a cell with room for three: three are written. results = await asyncio.gather( - *(directory.place(f"{q}-{i}", q, _never) for i in range(10)), return_exceptions=True + *(directory.place(f"{q}-{i}", q, _never, actor="operator") for i in range(10)), + return_exceptions=True, ) assert sum(isinstance(r, tuple) for r in results) == 3 assert sum(isinstance(r, directory.PlacementRefused) for r in results) == 7 @@ -145,10 +147,13 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: assert held == 3 # An org placed already answers its cell, whatever the quota or the cell's state. placed = next(f"{q}-{i}" for i, r in enumerate(results) if isinstance(r, tuple)) - assert await directory.place(placed, q, lambda _r: "closed") == (q, f"dep-{q}") + assert await directory.place(placed, q, lambda _r: "closed", actor="operator") == ( + q, + f"dep-{q}", + ) # A refusal judged on the locked row writes nothing. with pytest.raises(directory.PlacementRefused, match="closed"): - await directory.place(f"new-{a}", a, lambda _r: "closed") + await directory.place(f"new-{a}", a, lambda _r: "closed", actor="operator") assert await directory.lookup(f"new-{a}") is None # Routing reads every cell with its upstream address ('' until attach writes it). cells, _aliases = await directory.routing() @@ -738,7 +743,9 @@ async def _held(*a: Any) -> None: conn = await asyncpg.connect(own_dsn) try: async with asyncio.timeout(30): - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await inserted.wait() batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme", "beta"])) await _waits_on_the_row_lock(conn, "FOR UPDATE") @@ -782,12 +789,17 @@ async def _held(cell_name: str) -> Any: async with asyncio.timeout(30): batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme"])) await written.wait() - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await batch == (1, 1) assert await live == (name, dep) - assert await _placements_of(conn, name) == (1, [("operator", "acme")]) + assert await _placements_of(conn, name) == ( + 1, + [("operator", "acme"), (f"cell:{name}", "acme")], + ) finally: release.set() await conn.close() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 4e45db4285..40ef634062 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", - "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", + "usage: loom cell [-h] {credential,limit,which,remove,place,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,place,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -455,6 +455,10 @@ "usage: loom cell remove [-h] --cell CELL [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL The cell to remove (its LOOM_CELL) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell place": [ + "usage: loom cell place [-h] --cell CELL --org ORG [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --org ORG The org to place --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom cell prepare": [ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index a31a862106..37cebe51bb 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12811,6 +12811,99 @@ } ] }, + "place": { + "prog": "loom cell place", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_place:cmd_place" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "org", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--org" + ], + "dest": "org", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The org to place", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] + }, "prepare": { "prog": "loom cell prepare", "usage": null, @@ -13163,6 +13256,7 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", + "place": "loom cell place", "prepare": "loom cell prepare", "attach": "loom cell attach" }, @@ -13187,6 +13281,11 @@ "help": "Remove the directory row of a cell attaching with no placement.", "metavar": "remove" }, + { + "dest": "place", + "help": "Place an org in this cell and record its tenancy.", + "metavar": "place" + }, { "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py new file mode 100644 index 0000000000..5f8deeaf0e --- /dev/null +++ b/tests/unit/test_cell_place.py @@ -0,0 +1,466 @@ +"""Operator placement keeps the directory winner and bounds its local session.""" + +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import asynccontextmanager +from dataclasses import asdict +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.unit.test_router import _row + +OPERATOR = "lco_" + "7" * 128 + + +@pytest.fixture(autouse=True) +def _code_org_reader(monkeypatch): + monkeypatch.setattr(cell, "_code_orgs", None) + + +class DirectoryConnection: + def __init__(self, winner: str | None = "eu-1") -> None: + self.winner = winner + self.row = _row() + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + self.committed = False + + @asynccontextmanager + async def transaction(self): + yield + self.committed = True + + async def execute(self, sql, *args): + self.calls.append((sql, args)) + + async def fetchrow(self, sql, *args): + self.calls.append((sql, args)) + if "FROM loom_router.cells WHERE" in sql: + return asdict(self.row) + if self.winner is not None: + return {"cell": self.winner, "deployment_id": f"dep-{self.winner}"} + return None + + async def fetchval(self, sql, *args): + self.calls.append((sql, args)) + self.winner = args[1] + return args[0] + + +@pytest.fixture +def connected(monkeypatch): + conn = DirectoryConnection() + + @asynccontextmanager + async def connection(**kw): + yield conn + + monkeypatch.setattr(directory, "_connection", connection) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + return conn + + +def request(client, **over): + return client.post( + "/cells/v1/operator/cells/eu-1/place", + json={ + "org": "acme", + "deployment_id": "dep-eu-1", + "fingerprint": cell.as_value( + {"public_host": "eu-1.example.test", "deployment_id": "dep-eu-1"} + ), + **over, + }, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + +def test_existing_winner_emits_audit_wal_before_returning(connected): + client = TestClient(app.build_app(app.Router())) + result = request(client) + assert result.status_code == 200 + assert result.json() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + audit = [(sql, args) for sql, args in connected.calls if "INSERT INTO loom_router.audit" in sql] + assert len(audit) == 1 and audit[0][1] == ("operator", {"org": "acme", "cell": "eu-1"}) + assert connected.committed + assert not any("UPDATE loom_router.cells SET placed" in sql for sql, _ in connected.calls) + + +@pytest.mark.parametrize( + "posted", + [{"deployment_id": "other"}, {"fingerprint": {}}, {"org": "x" * (cell.ORG_MAX_BYTES + 1)}], +) +def test_existing_winner_refuses_operator_identity_mismatch(connected, posted): + result = request(TestClient(app.build_app(app.Router())), **posted) + assert result.status_code in (400, 409) + assert not connected.committed + assert not any("INSERT" in sql for sql, _ in connected.calls) + + +def test_operator_authentication_precedes_directory_access(connected): + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", json={} + ) + assert result.status_code == 401 and connected.calls == [] + + +def test_live_duplicate_also_emits_audit_wal(connected): + assert asyncio.run( + directory.place("acme", "eu-1", lambda row: "closed", actor="cell:eu-1") + ) == ("eu-1", "dep-eu-1") + assert connected.committed + assert any("INSERT INTO loom_router.audit" in sql for sql, _ in connected.calls) + + +class LocalConnection: + def __init__(self): + self.calls = [] + self.committed = False + self.terminated = False + self.stall_commit = False + self.stall_close = False + self.fail_insert = False + self.commit_started = asyncio.Event() + + def transaction(self): + conn = self + + class Transaction: + async def start(self): + pass + + async def commit(self): + conn.commit_started.set() + if conn.stall_commit: + await asyncio.Event().wait() + conn.committed = True + + return Transaction() + + async def execute(self, sql, *args): + self.calls.append(sql) + if "INSERT" in sql and self.fail_insert: + raise OSError("local insert failed") + return "INSERT 0 0" + + def terminate(self): + self.terminated = True + + +@pytest.fixture +def local(monkeypatch): + from server.cli import cell_place + from server.core import pg + + conn = LocalConnection() + state = {"first": "eu-1", "state": "enforced", "deployment": "dep-eu-1", "row": None} + + async def read_state(c): + return state["state"], state["first"] + + async def fetchval(sql): + return state["deployment"] + + async def fetchrow(sql): + return state["row"] + + conn.fetchval = fetchval + conn.fetchrow = fetchrow + + @asynccontextmanager + async def dedicated(**kw): + try: + yield conn + finally: + if conn.stall_close: + await asyncio.Event().wait() + + async def direct(c, what): + return [] + + async def take_lock(c, what): + conn.calls.append("lock") + + async def held(c): + return True + + async def directory_call(*a): + conn.calls.append("directory") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell, "read_state_if_present", read_state) + monkeypatch.setattr(cell_place.apg, "direct_refusals", direct) + monkeypatch.setattr(cell_place.apg, "take_lock", take_lock) + monkeypatch.setattr(cell_place.apg, "lock_held", held) + monkeypatch.setattr(cell_place, "_directory", directory_call) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.03) + fields = cell.fingerprint("dep-eu-1") + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + return conn, state + + +async def invoke_place(): + import argparse + + import httpx + + from server.cli import cell_place + + return await cell_place.place( + argparse.Namespace(cell="eu-1", org="acme"), httpx.URL("https://router.test") + ) + + +async def test_local_noop_insert_still_commits_durability_message(local): + from server.core import pg + + conn, state = local + assert await invoke_place() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + assert conn.committed and conn.terminated + assert conn.calls[0:2] == ["lock", "directory"] + assert conn.calls[-1] == pg._DURABLE_MESSAGE + + +@pytest.mark.parametrize( + "change", + [ + {"state": "recording"}, + {"state": "partial"}, + {"first": "us-1"}, + {"deployment": "other"}, + {"row": None}, + ], +) +async def test_local_invalid_identity_refuses_before_directory(local, change): + from server.cli import cell_place + + conn, state = local + state.update(change) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert conn.calls == ["lock"] and conn.terminated + + +@pytest.mark.parametrize("change", [{"state": "recording"}, {"first": "us-1"}, {"row": None}]) +async def test_local_identity_is_read_again_before_insert(local, monkeypatch, change): + from server.cli import cell_place + + conn, state = local + + async def changed(*a): + state.update(change) + + monkeypatch.setattr(cell_place, "_directory", changed) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert not any("INSERT" in sql for sql in conn.calls) + assert conn.terminated and not conn.committed + + +async def test_local_insert_failure_can_be_retried(local): + conn, state = local + conn.fail_insert = True + with pytest.raises(OSError, match="local insert failed"): + await invoke_place() + assert not conn.committed and conn.terminated + conn.fail_insert = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_and_cleanup_stalls_are_bounded(local): + conn, state = local + conn.stall_commit = conn.stall_close = True + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert conn.commit_started.is_set() and conn.terminated and not conn.committed + conn.stall_commit = conn.stall_close = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_cancellation_closes_the_session(local): + conn, state = local + conn.stall_commit = True + task = asyncio.create_task(invoke_place()) + await conn.commit_started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert conn.terminated and not conn.committed + + +async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): + from server.core import pg + + conn, state = local + cleaned = False + + @asynccontextmanager + async def stalled(**kw): + nonlocal cleaned + try: + await asyncio.Event().wait() + yield conn + finally: + cleaned = True + + monkeypatch.setattr(pg, "dedicated", stalled) + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert cleaned and conn.calls == [] + + +async def test_query_timeout_cannot_enter_an_unbounded_rollback(local): + conn, state = local + + class Transaction: + async def start(self): + pass + + async def commit(self): + pytest.fail("a stalled statement cannot commit") + + async def __aenter__(self): + await self.start() + + async def __aexit__(self, *exc): + await asyncio.Event().wait() + + async def stalled(sql, *args): + await asyncio.Event().wait() + + conn.transaction = Transaction + conn.execute = stalled + task = asyncio.create_task(invoke_place()) + try: + done, _ = await asyncio.wait([task], timeout=0.15) + assert done, "the statement deadline must include ending its transaction" + with pytest.raises(TimeoutError): + await task + assert conn.terminated + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize( + "status,answer", + [ + (200, {"org": "acme", "cell": "us-1", "deployment_id": "dep-us-1"}), + (200, {"org": "acme", "cell": "eu-1", "deployment_id": "wrong"}), + (200, []), + (200, {"org": "someone-else", "cell": "eu-1", "deployment_id": "dep-eu-1"}), + (503, {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"}), + (504, {"error": "gateway timeout"}), + ], +) +async def test_cli_refuses_unknown_response_or_other_winner(monkeypatch, status, answer): + import httpx + + from server.cli import cell_place + + async def respond(request): + assert request.headers["authorization"] == f"Bearer {settings.router_operator_token}" + return httpx.Response(status, json=answer) + + real = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real(transport=httpx.MockTransport(respond), **kw) + ) + with pytest.raises(cell_place.PlacementRefused): + await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +async def test_repository_only_configured_orgs_are_in_full_cli_fingerprint(local, monkeypatch): + from server.cli import cell_place + from server.code.domain.index import repos + + conn, state = local + monkeypatch.setattr(repos, "operator_orgs", lambda: {"repo-only"}) + fields = cell.fingerprint("dep-eu-1") + fields["configured_orgs"] = sorted(set(fields["configured_orgs"]) | {"repo-only"}) + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + _, value = await cell_place._identity(conn, "eu-1") + assert "repo-only" in value["fields"]["configured_orgs"] + + +def test_new_placement_uses_counted_statement_bound(monkeypatch): + from tests.unit.test_router_attach import _cell_row, _Conn, _Store + + store = _Store(_cell_row(state="serving")) + + @asynccontextmanager + async def connection(**kw): + yield _Conn(store) + + monkeypatch.setattr(directory, "_connection", connection) + asyncio.run( + directory.place("new", "eu-1", lambda row: "", actor="operator", verify=lambda row: "") + ) + assert store.statements == directory.PLACE_STATEMENTS + assert directory.PLACE_MAX_S == directory._bound_s(store.statements) + + +def test_operator_placement_body_bound_refuses_before_directory(connected): + from server.router.limits import MAX_BODY_BYTES + + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", + content=b"x" * (MAX_BODY_BYTES + 1), + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + assert result.status_code == 413 and connected.calls == [] + + +def test_operator_place_bypasses_cell_token_rate_limiter(connected, monkeypatch): + router = app.Router() + monkeypatch.setattr( + router.placements, "admit", lambda name: pytest.fail("operator used cell rate") + ) + assert request(TestClient(app.build_app(router))).status_code == 200 + + +@pytest.mark.parametrize("org", ["", "-", "*", "bad\x00org"]) +def test_operator_place_refuses_malformed_org_before_directory(connected, org): + assert request(TestClient(app.build_app(app.Router())), org=org).status_code == 400 + assert connected.calls == [] + + +def test_cli_cancellation_names_the_idempotent_retry_and_preserves_cancellation( + monkeypatch, capsys +): + import argparse + + import httpx + + from server.cli import cell_place + + cancelled = asyncio.CancelledError("commit response was cancelled") + + async def cancel(*args): + raise cancelled + + monkeypatch.setattr(cell_place, "_args_problem", lambda args: "") + monkeypatch.setattr( + cell_place.cell_cli, "operator_router", lambda args: httpx.URL("https://router.test") + ) + monkeypatch.setattr(cell_place, "place", cancel) + monkeypatch.setattr(cell_place, "run_connected", lambda run, **kw: asyncio.run(run(None))) + with pytest.raises(asyncio.CancelledError) as caught: + cell_place.cmd_place(argparse.Namespace(cell="eu-1", org="acme")) + assert caught.value is cancelled + printed = capsys.readouterr() + assert printed.out == "" + assert "placement may have committed; rerun the same command" in printed.err diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 9de51a1e75..ff59af4e14 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -82,7 +82,9 @@ async def first_cell_named(self, name: str) -> directory.Cell | None: row = self.cells.get(name) return row if row is not None and row.is_first else None - async def place(self, org: str, name: str, refusal: Any) -> tuple[str, str]: + async def place( + self, org: str, name: str, refusal: Any, *, actor: str, verify: Any = None + ) -> tuple[str, str]: self._check() if org not in self.placements: row = self.cells[name] # the row as the placement locks it diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index efb0b29e80..bab0420cc4 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -900,7 +900,7 @@ def sent(call: Any) -> int: def test_a_live_placement_reads_the_winner_again_under_the_row_lock(store: _Store) -> None: """A batch holds the cell's row lock, not the org's: an org it placed between the live placement's first read and its row lock is answered placed, never judged on the quota, - counted or audited twice.""" + counted twice. Its call still writes the durability audit.""" store.cells["eu-1"].update(state="serving", placements_max=0) reads = 0 real = _Conn.fetchrow @@ -915,9 +915,10 @@ async def _fetchrow(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchrow", _fetchrow) - winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) + winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) assert winner == ("eu-1", "dep-1") and reads == 2 - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] def test_a_live_insert_that_loses_to_the_primary_key_counts_nothing(store: _Store) -> None: @@ -931,8 +932,12 @@ async def _fetchval(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchval", _fetchval) - assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) == ("eu-1", "dep-1") - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) == ( + "eu-1", + "dep-1", + ) + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] # The attach lock and the troubleshooting page. From 042dd5e42c27680d8bc9c198101c71dc1733191e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:41:50 +0000 Subject: [PATCH 17/25] Prove cell placement retries survive standby promotion Exercise primary-visible commits, durable duplicate retries and cancellation cleanup on an isolated physical standby pair in both supported Postgres CI jobs. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 29 ++ .../integration/test_cell_place_durability.py | 310 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 tests/integration/test_cell_place_durability.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22342c2cab..702540d1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -899,7 +899,36 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_place.py tests/integration/test_cell_org_triggers_pg.py + - name: pytest placement durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-place-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_CELL_PLACE_DRILL: "1" + LOOM_CELL_PLACE_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-place-drill-pg loom-scale-place-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-place-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-place-drill-pg loom-scale-place-drill-pg-standby || true + docker network rm loom-scale-place-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-place-drill-pg loom-scale-place-drill-pg-standby + pytest -q tests/integration/test_cell_place_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/tests/integration/test_cell_place_durability.py b/tests/integration/test_cell_place_durability.py new file mode 100644 index 0000000000..1d788f47d6 --- /dev/null +++ b/tests/integration/test_cell_place_durability.py @@ -0,0 +1,310 @@ +"""Placement retries survive promotion of a private physical standby. + +Only the dedicated CI drill enables this file. Its pair is separate from the +integration server's stores because it pauses replication and stops the primary. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import os +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlsplit + +import asyncpg +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.core.schema import pg_migrate +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-place-drill-pg" +STANDBY = PRIMARY + "-standby" +ORG = "placement-durability" +DEPLOYMENT = "placement-drill" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +@dataclass +class Drill: + observer: Any + directory_dsn: str + local_dsn: str + tasks: list[asyncio.Task] = field(default_factory=list) + paused: bool = False + + def task(self, work: Any) -> asyncio.Task: + task = asyncio.create_task(work) + self.tasks.append(task) + return task + + async def pause(self) -> None: + await _docker("pause", STANDBY) + self.paused = True + + async def resume(self) -> None: + await _docker("unpause", STANDBY) + self.paused = False + + async def waiting(self, task: asyncio.Task, application: str) -> int: + async with asyncio.timeout(5): + while True: + assert not task.done(), "placement acknowledged before the standby resumed" + pid = await self.observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + application, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.02) + + async def seed(self, dsn: str, sql: str, *, marked: bool = False) -> None: + """Cancel COMMIT only after it is primary-visible and waiting for standby flush.""" + application = f"placement-seed-{len(self.tasks)}" + conn = await asyncpg.connect( + dsn, + timeout=5, + command_timeout=5, + server_settings={ + "application_name": application, + "synchronous_commit": "on", + **(pg.marker_settings() if marked else {}), + }, + ) + try: + await conn.execute("BEGIN") + await conn.execute(sql) + commit = self.task(conn.execute("COMMIT")) + await self.waiting(commit, application) + commit.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(commit, 5) + finally: + conn.terminate() + + +@pytest.fixture +async def drill(monkeypatch): + if not os.environ.get("LOOM_CELL_PLACE_DRILL"): + pytest.skip("LOOM_CELL_PLACE_DRILL not set: requires its disposable physical standby") + assert os.environ["LOOM_CELL_PLACE_DRILL"] == "1" + dsn = os.environ["LOOM_CELL_PLACE_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + state = None + try: + # Refuse to disrupt a container unless the explicit DSN identifies this exact pair. + identifier = str( + await observer.fetchval("SELECT system_identifier FROM pg_control_system()") + ) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + _, directory_dsn = await create_database(dsn) + _, local_dsn = await create_database(dsn) + state = Drill(observer, directory_dsn, local_dsn) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(settings, "router_pg_dsn", directory_dsn) + monkeypatch.setattr(settings, "pg_dsn", local_dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_statement_timeout_ms", 10_000) + previous_expected = pg._expected + previous_orgs = cell._code_orgs + pg.expect_deployment(None) + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + try: + async with asyncio.timeout(60): + await _initialize(state) + yield state + finally: + pg.expect_deployment(previous_expected) + cell._code_orgs = previous_orgs + finally: + if state is not None: + if state.paused: + await state.resume() + for task in state.tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(15): + await asyncio.gather(*state.tasks, return_exceptions=True) + await directory.close() + await pg.close_pool() + observer.terminate() + + +async def _initialize(drill: Drill) -> None: + @contextlib.asynccontextmanager + async def connect(): + conn = await asyncpg.connect(drill.local_dsn, command_timeout=10) + try: + yield conn + finally: + conn.terminate() + + await pg_migrate.migrate(connect=connect) + await directory.migrate() + fingerprint = cell.as_value(cell.fingerprint(DEPLOYMENT)) + async with connect() as conn: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + DEPLOYMENT, + ) + await conn.execute("INSERT INTO loom.cell_first (is_first, cell) VALUES (true, 'first')") + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", json.dumps(fingerprint) + ) + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ('first', true, 'first.example.test', 'serving', 'enforced', $1, $2, $3)", + "3" * 64, + DEPLOYMENT, + fingerprint, + ) + + +async def _directory_retry(drill: Drill) -> None: + await drill.pause() + await drill.seed( + drill.directory_dsn, + "INSERT INTO loom_router.placements (org, cell) VALUES ('placement-durability', 'first'); " + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = 'first'", + ) + assert await directory.lookup(ORG) == ("first", "first.example.test") + retry = drill.task(directory.place(ORG, "first", lambda _: "", actor="operator")) + await drill.waiting(retry, "loom-router") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == ("first", DEPLOYMENT) + + +async def _local_retry(drill: Drill, monkeypatch) -> None: + async def verified_directory(router, name, org, deployment, fingerprint): + # The directory retry above already acknowledged this winner's physical flush. + assert (name, org, deployment) == ("first", ORG, DEPLOYMENT) + assert await directory.lookup(org) == (name, "first.example.test") + + monkeypatch.setattr(cell_place, "_directory", verified_directory) + await drill.pause() + await drill.seed( + drill.local_dsn, + "INSERT INTO loom.tenancy (org) VALUES ('placement-durability')", + marked=True, + ) + assert ( + await _sql( + PRIMARY, + urlsplit(drill.local_dsn).path.removeprefix("/"), + "SELECT org FROM loom.tenancy", + ) + == ORG + ) + args = argparse.Namespace(cell="first", org=ORG) + cancelled = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + pid = await drill.waiting(cancelled, "loom") + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(cancelled, cell_place._bound_s() + 1) + async with asyncio.timeout(5): + while True: + if not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.02) + assert not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ) + retry = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + await drill.waiting(retry, "loom") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == { + "org": ORG, + "cell": "first", + "deployment_id": DEPLOYMENT, + } + + +@pytest.mark.recovery +@pytest.mark.destructive +async def test_duplicate_placement_survives_physical_standby_promotion(drill, monkeypatch): + async with asyncio.timeout(120): + await _directory_retry(drill) + await _local_retry(drill, monkeypatch) + await directory.close() + await pg.close_pool() + drill.observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + directory_db = urlsplit(drill.directory_dsn).path.removeprefix("/") + local_db = urlsplit(drill.local_dsn).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT p.org, p.cell, c.deployment_id, c.placed FROM loom_router.placements p " + "JOIN loom_router.cells c USING (cell)", + ) + == f"{ORG}|first|{DEPLOYMENT}|1" + ) + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT actor, action, detail->>'org', detail->>'cell' FROM loom_router.audit", + ) + == f"operator|place|{ORG}|first" + ) + assert await _sql(STANDBY, local_db, "SELECT org FROM loom.tenancy") == ORG From d27ff2e5e39bcdbd2c61e3d541a01a50d2e13d14 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:04:24 +0000 Subject: [PATCH 18/25] Close placement sessions before cancelling their work Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 +- server/cli/cell_control.py | 80 +++-- server/cli/cell_place.py | 52 +-- server/router/app.py | 2 +- .../integration/test_cell_control_cleanup.py | 12 +- tests/integration/test_cell_place_cleanup.py | 106 ++++++ tests/unit/test_cell_attach_cleanup.py | 10 +- tests/unit/test_cell_control_census.py | 8 +- tests/unit/test_cell_place.py | 164 +++++++++- tests/unit/test_cell_place_owner.py | 307 ++++++++++++++++++ 10 files changed, 672 insertions(+), 78 deletions(-) create mode 100644 tests/integration/test_cell_place_cleanup.py create mode 100644 tests/unit/test_cell_place_owner.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 4cdb16ecaf..379e91853c 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -764,8 +764,13 @@ read-only scope, lists any tenancy row the directory does not place. session, requires a full enforced local cell state and the full accepted configuration fingerprint, and checks both again before inserting. A transactional durability message makes even a no-op insert flush an earlier primary-only tenancy row to the standby. - Opening the session, local statements, COMMIT and cleanup have client deadlines derived - from `LOOM_PG_STATEMENT_TIMEOUT_MS`. An unknown outcome names rerunning the same command; + Opening the session, initial checks and the local transaction through COMMIT each have + a separate client deadline derived from `LOOM_PG_STATEMENT_TIMEOUT_MS`; the directory + request keeps its own transport timeout. Each local phase reserves cleanup inside its + original deadline. On failure the shared control owner closes the exact backend before + cancelling or reaping its work, deferring repeated signals within that same deadline. + Successful exit and failures outside a local phase have a separate cleanup allowance + derived from the same setting. An unknown outcome names rerunning the same command; the permanent directory winner and idempotent local insert let that rerun finish. The route checks directory state; a direct operator API caller is trusted to check local Postgres mode as the CLI does. For an org the directory does not place yet it diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index c4360839ed..74d804fdb6 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -119,7 +119,8 @@ async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: class Owner: """One raw backend and one cleanup owner for a native control command.""" - def __init__(self) -> None: + def __init__(self, *, exit_s: float = pg._RELEASE_TIMEOUT_S) -> None: + self.exit_s = exit_s self.conn: Any = None self.phase_end: float | None = None self.close_end: float | None = None @@ -155,7 +156,7 @@ async def _open(self, what: str) -> None: async def close(self) -> None: if not self.closed: self.closed = True - self.close_end = self.phase_end or (time.monotonic() + pg._RELEASE_TIMEOUT_S) + self.close_end = self.phase_end or (time.monotonic() + self.exit_s) if self.conn is None: return if self._close_task is None: @@ -163,12 +164,17 @@ async def close(self) -> None: _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) ) self._close_task.add_done_callback(pg._retrieve_task) + interrupted: asyncio.CancelledError | None = None try: - done, _ = await asyncio.wait( - {self._close_task}, timeout=max(0, cast(float, self.close_end) - time.monotonic()) - ) - if not done: - raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + while not self._close_task.done(): + remaining = cast(float, self.close_end) - time.monotonic() + if remaining <= 0: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + try: + await asyncio.wait({self._close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Signals cannot abort the public cancellation already in flight. + interrupted = interrupted or error result = self._close_task.result() if isinstance(result, _Fatal): raise _ForwardFatal(result) @@ -176,7 +182,11 @@ async def close(self) -> None: pg._terminate(self.conn) if not self._close_task.done(): self._close_task.cancel() + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: for task in tasks: @@ -219,31 +229,43 @@ async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) + async def run[T]( + self, + opening: Callable[[], Awaitable[None]], + work: Callable[[], Awaitable[T]], + *, + opening_s: float, + ) -> T: + """Own one command; fatal guard failures leave only after bounded cleanup.""" + if self.conn is not None or self.phase_end is not None or self._tasks or self.closed: + raise _refused("a command needs a fresh control owner") + try: + await self.call(opening, total_s=opening_s) + result = await asyncio.shield(self._start(work)) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await self.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await self._stop(set(self._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise + try: + await self.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None + finally: + await self._stop(set(self._tasks)) + return result + async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: - """Run at the command boundary; fatal guard exceptions leave only after cleanup.""" + """Run native control with its existing opening, transaction and exit allowances.""" owner = Owner() - try: - await owner.call(lambda: owner._open(what), total_s=OPEN_S) - result = await asyncio.shield(owner._start(lambda: work(owner))) - if isinstance(result, _Fatal): - raise _ForwardFatal(result) - except BaseException as original: - try: - await owner.close() - except BaseException: # noqa: BLE001 — preserve the command's original failure. - pass - await owner._stop(set(owner._tasks)) - if isinstance(original, _ForwardFatal): - raise original.fatal.error from None - raise - try: - await owner.close() - except _ForwardFatal as interrupted: - raise interrupted.fatal.error from None - finally: - await owner._stop(set(owner._tasks)) - return result + return await owner.run(lambda: owner._open(what), lambda: work(owner), opening_s=OPEN_S) async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index 3820ecf2cc..573e0ff2fc 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -14,6 +14,7 @@ from server.cli import cell_attach_pg as apg from server.cli._ch import run_connected, writes from server.cli.cell_ch import Refused +from server.cli.cell_control import Owner from server.core import cell, pg from server.router.directory import PLACE_MAX_S from server.settings import settings @@ -88,18 +89,34 @@ async def _directory( async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: """Hold the operator lock through directory placement and an acknowledged local commit. Every local phase, including opening and closing the direct session, has a deadline.""" - cm = pg.dedicated(statement_timeout_ms=settings.pg_statement_timeout_ms) - conn = None - try: - async with asyncio.timeout(_bound_s()): - conn = await cm.__aenter__() - async with asyncio.timeout(_bound_s()): - if refusals := await apg.direct_refusals(conn, "place"): - raise Refused(refusals) - await apg.take_lock(conn, "place") - deployment, fingerprint = await _identity(conn, args.cell) + owner = Owner(exit_s=_bound_s()) + + async def opening() -> None: + owner.conn = await pg._connect_dedicated( + statement_timeout_ms=settings.pg_statement_timeout_ms + ) + if owner.closed: + await owner.close() + raise PlacementRefused("connection arrived after the placement attempt ended") + try: + await pg._check_deployment(owner.conn) + await pg.init_connection(owner.conn) + except Exception as error: + pg._raise_if_unavailable(error) + raise + + async def initial() -> tuple[str, dict[str, Any]]: + if refusals := await apg.direct_refusals(owner.conn, "place"): + raise Refused(refusals) + await apg.take_lock(owner.conn, "place") + return await _identity(owner.conn, args.cell) + + async def work() -> dict[str, str]: + deployment, fingerprint = await owner.call(initial, total_s=_bound_s()) await _directory(router, args.cell, args.org, deployment, fingerprint) - async with asyncio.timeout(_bound_s()): + + async def local() -> None: + conn = owner.conn tx = conn.transaction() await tx.start() if not await apg.lock_held(conn): @@ -114,14 +131,11 @@ async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: # A no-op insert must also flush a primary-only predecessor to the standby. await conn.execute(pg._DURABLE_MESSAGE) await tx.commit() - finally: - if conn is not None: - try: - async with asyncio.timeout(_bound_s()): - await cm.__aexit__(None, None, None) - finally: - conn.terminate() - return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + await owner.call(local, total_s=_bound_s()) + return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + return await owner.run(opening, work, opening_s=_bound_s()) def cmd_place(args: argparse.Namespace) -> int: diff --git a/server/router/app.py b/server/router/app.py index 6cf6d9e729..212e87f737 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -320,7 +320,7 @@ def refusal(locked: directory.Cell) -> str: async def operator_place(self, request: Request) -> JSONResponse: """Place with the operator credential. The CLI checks its local Postgres mode; this route trusts that check and verifies the directory's locked identity.""" - body = await self._operator_body(request) + body = await self._operator_body(request, MAX_BODY_BYTES) if isinstance(body, JSONResponse): return body org = cell_state.named_org(body.get("org")) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index e057fef323..8f68939b62 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -105,6 +105,8 @@ async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( real_close = asyncpg.Connection.close observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() backend = {} + close_interrupted = asyncio.Event() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) async def read(conn, **_): backend["conn"], backend["pid"] = conn, conn.get_server_pid() @@ -115,7 +117,11 @@ async def delayed_close(self, **kwargs): await real_close(self, **kwargs) if self is backend.get("conn"): close_started.set() - await close_return.wait() + try: + await close_return.wait() + except asyncio.CancelledError: + close_interrupted.set() + raise monkeypatch.setattr(cell_prepare, "_read_rows", read) monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) @@ -128,6 +134,10 @@ async def delayed_close(self, **kwargs): task.cancel("first native cancellation") await asyncio.wait_for(close_started.wait(), 1) task.cancel("second cancellation") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not close_interrupted.is_set(), "second signal cannot interrupt public close" + assert not task.done() with pytest.raises(asyncio.CancelledError, match="first native cancellation"): await asyncio.wait_for(task, 0.5) await released(native_database, backend["pid"]) diff --git a/tests/integration/test_cell_place_cleanup.py b/tests/integration/test_cell_place_cleanup.py new file mode 100644 index 0000000000..3bbb5acc07 --- /dev/null +++ b/tests/integration/test_cell_place_cleanup.py @@ -0,0 +1,106 @@ +"""Placement cleanup observes the actual marked backend through interrupted work.""" + +from __future__ import annotations + +import asyncio +import contextlib +import time + +import asyncpg +import pytest + +from server.cli import cell_place +from server.core import pg +from tests.integration import test_cell_place as placement + + +@pytest.fixture +async def stores(monkeypatch): + async for value in placement.stores.__wrapped__(monkeypatch): + yield value + + +async def waiting(observer, pid): + async with asyncio.timeout(3): + while True: + if await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event='PgSleep')", + pid, + ): + return + await asyncio.sleep(0.01) + + +async def assert_released(observer, pid): + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +@pytest.mark.parametrize("interrupt", ["deadline", "cancel", "second_cancel"]) +async def test_placement_owner_closes_before_cancelling_actual_rollback( + stores, monkeypatch, interrupt +): + # Only the exact ROLLBACK wire operation is delayed. The transaction context, + # sleeping query, session lock, observer and close are the installed real driver. + original_identity, execute = cell_place._identity, asyncpg.Connection.execute + known = asyncio.Event() + captured = {} + queries = [] + + async def stalled_rollback(conn, sql, *args, **kwargs): + if conn is captured.get("conn") and sql == "ROLLBACK;": + query = asyncio.create_task(execute(conn, "SELECT pg_sleep(10)")) + queries.append(query) + while not query.done(): + try: + await asyncio.shield(query) + except asyncio.CancelledError: + continue + query.result() + return await execute(conn, sql, *args, **kwargs) + + async def identity(conn, name): + await original_identity(conn, name) + captured["conn"] = conn + known.set() + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(193789)") + raise ValueError("placement body failed before rollback") + + monkeypatch.setattr(cell_place, "_identity", identity) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + observer = await asyncpg.connect(stores["first"]) + task = asyncio.create_task(placement._local(stores, monkeypatch, "first")) + try: + await asyncio.wait_for(known.wait(), 3) + conn = captured["conn"] + pid = conn.get_server_pid() + await waiting(observer, pid) + started = time.monotonic() + if interrupt != "deadline": + task.cancel() + if interrupt == "second_cancel": + # A second signal arrives while the same operation is still active. + await asyncio.sleep(0) + task.cancel() + done, _ = await asyncio.wait({task}, timeout=0.8) + assert done, "placement must close its backend before waiting for rollback unwind" + with pytest.raises(TimeoutError if interrupt == "deadline" else asyncio.CancelledError): + await task + assert time.monotonic() - started < 0.8 + await assert_released(observer, pid) + finally: + # Exact owned test resources only; even a deliberately failing old owner + # cannot leave its query or scratch database alive after the regression. + if "conn" in captured and not captured["conn"].is_closed(): + with contextlib.suppress(Exception): + await captured["conn"].close(timeout=1) + if not task.done(): + task.cancel() + await asyncio.gather(task, *queries, return_exceptions=True) + await observer.close(timeout=1) + pg.expect_deployment(None) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index 11bf1bf094..a018784d21 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -8,7 +8,6 @@ import pytest from server.cli import cell_attach, cell_control -from server.core import pg from tests.unit.cell_control_owner import owner from tests.unit.test_cell_control import snapshot @@ -52,6 +51,9 @@ async def close(self, **kwargs): def terminate(self): events.append("abort") + def is_closed(self): + return False + async def work(): entered.set() try: @@ -78,7 +80,7 @@ async def work(): @pytest.mark.asyncio -async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(monkeypatch): +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() conn.terminate = lambda: None @@ -88,8 +90,8 @@ async def close(**_): await asyncio.Future() conn.close.side_effect = close - owned = owner(conn) - monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + owned = cell_control.Owner(exit_s=0.03) + owned.conn = conn started = time.monotonic() with pytest.raises(TimeoutError): await owned.close() diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 45b8e85baa..7fc1fb2e2b 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -143,7 +143,7 @@ def test_cell_control_migrations_are_classified(): ("register", "INSERT INTO", "loom_router.audit"): 1, ("_place_one", "INSERT INTO", "loom_router.placements"): 1, ("_place_one", "UPDATE", "loom_router.cells"): 1, - ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("place", "INSERT INTO", "loom_router.audit"): 1, ("_audit", "INSERT INTO", "loom_router.audit"): 1, }, } @@ -493,9 +493,9 @@ def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): # Operator reads may audit, but cannot bypass control guards for a state write. assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) - assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( - found.functions["place"] - ) + assert "_place_one(conn, org, cell_name, candidate, refusal)" in calls(found.functions["place"]) + assert "_cell(locked)" in calls(found.functions["place"]) + assert "verify(candidate)" in calls(found.functions["place"]) assert "refusal(candidate)" in calls(found.functions["_place_one"]) for name in ("set_limit", "rotate_credential"): assert not any(call.startswith("claim(") for call in calls(found.functions[name])) diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py index 5f8deeaf0e..26a06748a1 100644 --- a/tests/unit/test_cell_place.py +++ b/tests/unit/test_cell_place.py @@ -125,6 +125,7 @@ def __init__(self): self.calls = [] self.committed = False self.terminated = False + self.closed = False self.stall_commit = False self.stall_close = False self.fail_insert = False @@ -151,9 +152,18 @@ async def execute(self, sql, *args): raise OSError("local insert failed") return "INSERT 0 0" + async def close(self, **kwargs): + self.calls.append("close") + if self.stall_close: + await asyncio.Event().wait() + self.closed = True + def terminate(self): self.terminated = True + def is_closed(self): + return self.closed or self.terminated + @pytest.fixture def local(monkeypatch): @@ -175,13 +185,12 @@ async def fetchrow(sql): conn.fetchval = fetchval conn.fetchrow = fetchrow - @asynccontextmanager - async def dedicated(**kw): - try: - yield conn - finally: - if conn.stall_close: - await asyncio.Event().wait() + async def connect(**kw): + assert kw == {"statement_timeout_ms": settings.pg_statement_timeout_ms} + return conn + + async def initialize(c): + assert c is conn async def direct(c, what): return [] @@ -195,7 +204,9 @@ async def held(c): async def directory_call(*a): conn.calls.append("directory") - monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(pg, "_check_deployment", initialize) + monkeypatch.setattr(pg, "init_connection", initialize) monkeypatch.setattr(cell, "read_state_if_present", read_state) monkeypatch.setattr(cell_place.apg, "direct_refusals", direct) monkeypatch.setattr(cell_place.apg, "take_lock", take_lock) @@ -224,9 +235,9 @@ async def test_local_noop_insert_still_commits_durability_message(local): conn, state = local assert await invoke_place() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} - assert conn.committed and conn.terminated + assert conn.committed and conn.closed and not conn.terminated assert conn.calls[0:2] == ["lock", "directory"] - assert conn.calls[-1] == pg._DURABLE_MESSAGE + assert conn.calls[-2:] == [pg._DURABLE_MESSAGE, "close"] @pytest.mark.parametrize( @@ -246,7 +257,7 @@ async def test_local_invalid_identity_refuses_before_directory(local, change): state.update(change) with pytest.raises(cell_place.PlacementRefused): await invoke_place() - assert conn.calls == ["lock"] and conn.terminated + assert conn.calls == ["lock", "close"] and conn.closed and not conn.terminated @pytest.mark.parametrize("change", [{"state": "recording"}, {"first": "us-1"}, {"row": None}]) @@ -262,7 +273,7 @@ async def changed(*a): with pytest.raises(cell_place.PlacementRefused): await invoke_place() assert not any("INSERT" in sql for sql in conn.calls) - assert conn.terminated and not conn.committed + assert conn.closed and not conn.terminated and not conn.committed async def test_local_insert_failure_can_be_retried(local): @@ -270,7 +281,7 @@ async def test_local_insert_failure_can_be_retried(local): conn.fail_insert = True with pytest.raises(OSError, match="local insert failed"): await invoke_place() - assert not conn.committed and conn.terminated + assert not conn.committed and conn.closed and not conn.terminated conn.fail_insert = False await invoke_place() assert conn.committed @@ -295,7 +306,7 @@ async def test_local_commit_cancellation_closes_the_session(local): task.cancel() with pytest.raises(asyncio.CancelledError): await task - assert conn.terminated and not conn.committed + assert conn.closed and not conn.terminated and not conn.committed async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): @@ -304,16 +315,14 @@ async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): conn, state = local cleaned = False - @asynccontextmanager async def stalled(**kw): nonlocal cleaned try: await asyncio.Event().wait() - yield conn finally: cleaned = True - monkeypatch.setattr(pg, "dedicated", stalled) + monkeypatch.setattr(pg, "_connect_dedicated", stalled) with pytest.raises(TimeoutError): await asyncio.wait_for(invoke_place(), 0.5) assert cleaned and conn.calls == [] @@ -346,7 +355,7 @@ async def stalled(sql, *args): assert done, "the statement deadline must include ending its transaction" with pytest.raises(TimeoutError): await task - assert conn.terminated + assert conn.closed and not conn.terminated finally: if not task.done(): task.cancel() @@ -464,3 +473,122 @@ async def cancel(*args): printed = capsys.readouterr() assert printed.out == "" assert "placement may have committed; rerun the same command" in printed.err + + +async def test_local_phases_remain_separate_from_directory_and_exit(local, monkeypatch): + from server.cli import cell_control, cell_place + + observed = [] + actual_call = cell_control.Owner.call + + async def timed(self, work, *, total_s): + observed.append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def directory_call(*args): + # HTTP owns its existing independent transport limit, not a local phase cap. + await asyncio.sleep(0.06) + + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(cell_place, "_directory", directory_call) + await invoke_place() + assert observed == [0.03, 0.03, 0.03] + conn, _ = local + assert conn.committed and conn.closed and not conn.terminated + + +@pytest.mark.parametrize("where", ["initialization", "initial", "transaction", "directory"]) +async def test_original_error_survives_failed_required_close(local, monkeypatch, where): + from server.cli import cell_place + from server.core import pg + + original = ValueError("original placement failure") + conn, _ = local + + async def fail(*args): + raise original + + async def bad_close(**kwargs): + raise RuntimeError("close failed") + + conn.close = bad_close + if where == "initialization": + monkeypatch.setattr(pg, "init_connection", fail) + elif where == "initial": + monkeypatch.setattr(cell_place, "_identity", fail) + elif where == "directory": + monkeypatch.setattr(cell_place, "_directory", fail) + else: + conn.execute = fail + with pytest.raises(ValueError) as caught: + await invoke_place() + assert caught.value is original + assert conn.terminated and not conn.committed + + +async def test_success_refuses_when_required_close_fails(local): + conn, _ = local + + async def fail(**kwargs): + raise OSError("required close failed") + + conn.close = fail + with pytest.raises(OSError, match="required close failed"): + await invoke_place() + assert conn.committed and conn.terminated + + +async def test_lost_original_lock_refuses_without_reconnecting(local, monkeypatch): + from server.cli import cell_place + + conn, _ = local + + async def lost(c): + assert c is conn + return False + + monkeypatch.setattr(cell_place.apg, "lock_held", lost) + with pytest.raises(cell_place.PlacementRefused, match="lock was lost"): + await invoke_place() + assert conn.calls == ["lock", "directory", "close"] + assert conn.closed and not conn.committed + + +@pytest.mark.parametrize("inside_reserve", [False, True]) +async def test_late_placement_connection_never_reaches_body(local, monkeypatch, inside_reserve): + from server.core import pg + + conn, _ = local + released, discarded = asyncio.Event(), asyncio.Event() + real_terminate = conn.terminate + real_close = conn.close + + def terminate(): + real_terminate() + discarded.set() + + async def close(**kw): + await real_close(**kw) + discarded.set() + + async def late(**kwargs): + try: + await asyncio.Future() + except asyncio.CancelledError: + if not inside_reserve: + while not released.is_set(): + try: + await released.wait() + except asyncio.CancelledError: + pass + return conn + + conn.terminate, conn.close = terminate, close + monkeypatch.setattr(pg, "_connect_dedicated", late) + with pytest.raises(TimeoutError): + await invoke_place() + released.set() + await asyncio.wait_for(discarded.wait(), 0.2) + assert conn.calls == (["close"] if inside_reserve else []) + assert conn.closed if inside_reserve else conn.terminated + assert not conn.committed diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py new file mode 100644 index 0000000000..1b8cd967e7 --- /dev/null +++ b/tests/unit/test_cell_place_owner.py @@ -0,0 +1,307 @@ +"""Placement reuses the control owner without replacing its phase budgets.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control +from server.core import pg +from tests.unit.test_cell_attach import FakeConn + +pytestmark = pytest.mark.asyncio + + +async def test_custom_exit_is_used_only_outside_a_local_phase(): + owner = cell_control.Owner(exit_s=0.09) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + before = time.monotonic() + assert await owner.run(opening, AsyncMock(return_value=17), opening_s=0.3) == 17 + assert before + 0.09 <= owner.close_end <= time.monotonic() + 0.09 + assert conn.executed == ["CLOSE"] + + +@pytest.mark.parametrize("occupied", ["conn", "phase", "task", "closed", "repeated"]) +async def test_command_entry_refuses_reuse_before_any_callback(occupied): + owner = cell_control.Owner() + held = None + if occupied == "conn": + owner.conn = FakeConn() + elif occupied == "phase": + owner.phase_end = time.monotonic() + 1 + elif occupied == "task": + held = owner._start(lambda: asyncio.Future()) + elif occupied == "closed": + await owner.close() + else: + await owner.run(AsyncMock(), AsyncMock(), opening_s=0.1) + opening, body = AsyncMock(), AsyncMock() + try: + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(opening, body, opening_s=0.1) + opening.assert_not_called() + body.assert_not_called() + finally: + if held is not None: + held.cancel() + await asyncio.gather(held, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "body"]) +async def test_recursive_entry_cannot_replace_the_original_backend(stage): + owner = cell_control.Owner() + original = FakeConn() + other_open, other_body = AsyncMock(), AsyncMock() + + async def recurse(): + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(other_open, other_body, opening_s=0.1) + + async def opening(): + if stage == "opening": + await recurse() + owner.conn = original + + async def body(): + if stage == "body": + await recurse() + return 3 + + assert await owner.run(opening, body, opening_s=0.1) == 3 + other_open.assert_not_called() + other_body.assert_not_called() + assert owner.conn is original and original.executed == ["CLOSE"] + + +async def test_native_wrapper_keeps_its_existing_three_budgets(monkeypatch): + observed = {} + actual_run = cell_control.Owner.run + actual_call = cell_control.Owner.call + + async def wrapped_run(self, opening, work, *, opening_s): + observed.update(opening=opening_s, exit=self.exit_s) + return await actual_run(self, opening, work, opening_s=opening_s) + + async def wrapped_call(self, work, *, total_s): + observed.setdefault("phases", []).append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def opening(self, what): + assert what == "native" + self.conn = FakeConn() + + monkeypatch.setattr(cell_control.Owner, "run", wrapped_run) + monkeypatch.setattr(cell_control.Owner, "call", wrapped_call) + monkeypatch.setattr(cell_control.Owner, "_open", opening) + assert ( + await cell_control.run("native", lambda owner: owner.transaction(AsyncMock(return_value=7))) + == 7 + ) + assert observed == {"opening": 75.0, "exit": pg._RELEASE_TIMEOUT_S, "phases": [75.0, 60.0]} + + +async def test_failed_phase_keeps_its_end_instead_of_custom_exit(): + owner = cell_control.Owner(exit_s=99) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + async def body(): + return await owner.call(lambda: asyncio.Future(), total_s=0.04) + + before = time.monotonic() + with pytest.raises(TimeoutError): + await owner.run(opening, body, opening_s=0.1) + assert before < owner.close_end <= before + 0.1 + assert owner.close_end == owner.phase_end + assert conn.executed == ["CLOSE"] + + +async def test_public_close_defers_repeated_signals_until_acknowledged_within_original_end(): + events = [] + closing, finish = asyncio.Event(), asyncio.Event() + + class Connection: + async def close(self, **kwargs): + events.append("close") + closing.set() + await finish.wait() + events.append("ack") + + def terminate(self): + events.append("abort") + + def is_closed(self): + return "ack" in events + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + original_end = owner.close_end + task.cancel("first signal") + await asyncio.sleep(0) + task.cancel("second signal") + await asyncio.sleep(0) + assert not task.done(), "a second signal must not interrupt owned public close" + assert owner.close_end == original_end + finish.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert events == ["close", "ack"] + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "initial", "directory", "transaction", "close"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +async def test_placement_fatal_stays_inside_owner_until_public_close(stage, kind): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse, asyncio, sys + import httpx + from server.cli import cell_place + from server.core import pg + stage, kind = sys.argv[1:] + original = (SystemExit if kind == 'SystemExit' else KeyboardInterrupt)('original') + events = [] + class Transaction: + async def start(self): pass + async def commit(self): events.append('commit') + class Connection: + def transaction(self): return Transaction() + async def execute(self, *args): + if stage == 'transaction': raise original + async def close(self, **kw): + events.append('close') + if stage == 'close': raise original + def terminate(self): events.append('abort') + conn = Connection() + async def connect(**kw): return conn + async def initialize(c): + assert c is conn + if stage == 'opening': raise original + async def noop(*args): pass + async def direct(c, what): return [] + async def identity(c, name): + assert c is conn + if stage == 'initial': raise original + return 'dep', {} + async def directory(*args): + if stage == 'directory': raise original + async def held(c): return True + async def main(): + pg._connect_dedicated = connect + pg._check_deployment = initialize + pg.init_connection = noop + cell_place.apg.direct_refusals = direct + cell_place.apg.take_lock = noop + cell_place.apg.lock_held = held + cell_place._identity = identity + cell_place._directory = directory + try: + await cell_place.place(argparse.Namespace(cell='first', org='acme'), httpx.URL('http://router')) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 1, events + await asyncio.sleep(0) + print('original-after-public-close') + return + raise AssertionError('fatal became success') + asyncio.run(main()) + """ + ) + result = await asyncio.to_thread( + subprocess.run, + [sys.executable, "-c", program, stage, kind], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-public-close" in result.stdout + + +async def test_repeated_signals_cannot_extend_an_expired_public_close(): + closing = asyncio.Event() + conn = FakeConn() + + async def stalled(**kwargs): + closing.set() + await asyncio.Future() + + conn.close = stalled + owner = cell_control.Owner(exit_s=0.04) + owner.conn = conn + started = time.monotonic() + task = asyncio.create_task(owner.close()) + await closing.wait() + end = owner.close_end + + async def signals(): + while not task.done(): + task.cancel("first signal") + await asyncio.sleep(0.001) + + signal_task = asyncio.create_task(signals()) + try: + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "repeated signals cannot restart the original cleanup allowance" + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert time.monotonic() - started < 0.2 + assert owner.close_end == end + assert "TERMINATE" in conn.executed + finally: + signal_task.cancel() + await asyncio.gather(signal_task, task, return_exceptions=True) + + +async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): + closing, acknowledged = asyncio.Event(), asyncio.Event() + events = [] + + class Connection: + async def close(self, **kwargs): + closing.set() + await acknowledged.wait() + events.append("ack") + + def is_closed(self): + # An aborted transport is closed even while its cancellation is pending. + return True + + def terminate(self): + events.append("abort") + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + task.cancel("stop") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() and events == [] + acknowledged.set() + with pytest.raises(asyncio.CancelledError, match="stop"): + await task + assert events == ["ack"] + finally: + acknowledged.set() + await asyncio.gather(task, return_exceptions=True) From c66786efc4aa06bd600df19e39deb02e09561b20 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:33:54 +0000 Subject: [PATCH 19/25] Preserve native owner errors when fallback abort fails Signed-off-by: zlareb1 --- server/cli/cell_control.py | 15 ++- tests/unit/test_cell_control_abort.py | 185 ++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_cell_control_abort.py diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 74d804fdb6..1966b9e62b 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -179,9 +179,13 @@ async def close(self) -> None: if isinstance(result, _Fatal): raise _ForwardFatal(result) except BaseException: - pg._terminate(self.conn) - if not self._close_task.done(): - self._close_task.cancel() + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first close failure. + pass + finally: + if not self._close_task.done(): + self._close_task.cancel() if interrupted is not None: raise interrupted from None raise @@ -197,7 +201,10 @@ async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. if self.conn is not None: - pg._terminate(self.conn) + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — failed abort preserves the caller's failure. + pass async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: if self.closed or self.phase_end is not None: diff --git a/tests/unit/test_cell_control_abort.py b/tests/unit/test_cell_control_abort.py new file mode 100644 index 0000000000..890ddcba27 --- /dev/null +++ b/tests/unit/test_cell_control_abort.py @@ -0,0 +1,185 @@ +"""A failed fallback abort cannot replace the native owner's first failure.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control + +pytestmark = pytest.mark.asyncio +ERRORS = (ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) +ABORT_ERRORS = (RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) + + +async def outcome(work): + try: + result = await cell_control._captured(work) + except BaseException as error: # noqa: BLE001 — observe the identical native failure inertly. + return error + assert isinstance(result, cell_control._Fatal), "a failed required close cannot succeed" + return result.error + + +@pytest.mark.parametrize("first_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_direct_owner_keeps_first_close_failure_and_retrieves_child( + monkeypatch, first_kind, abort_kind +): + first, secondary = first_kind("first close failure"), abort_kind("abort failure") + aborted = [] + + class Connection: + def terminate(self): + aborted.append(True) + raise secondary + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + close = AsyncMock(side_effect=first) + monkeypatch.setattr(cell_control.pg, "close_owned", close) + result = await outcome(owner.close) + assert result is first + assert aborted == [True] + close.assert_awaited_once() + assert owner.closed and owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +@pytest.mark.parametrize("first_kind", ERRORS) +async def test_outer_signal_wins_over_later_close_and_abort_failures( + monkeypatch, first_kind, abort_kind +): + entered, finish = asyncio.Event(), asyncio.Event() + first, secondary = first_kind("close failed after signal"), abort_kind("abort failure") + seen = [] + actual_wait = asyncio.wait + owner = cell_control.Owner(exit_s=0.2) + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + await finish.wait() + raise first + + async def wait(*args, **kwargs): + try: + return await actual_wait(*args, **kwargs) + except asyncio.CancelledError as error: + seen.append(error) + raise + + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + monkeypatch.setattr(cell_control.asyncio, "wait", wait) + task = asyncio.create_task(outcome(owner.close)) + try: + await entered.wait() + original_end = owner.close_end + task.cancel("original outer cancellation") + await asyncio.sleep(0) + assert len(seen) == 1 and not task.done() + finish.set() + result = await task + assert result is seen[0] + assert owner.close_end == original_end + assert owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_abort_failure_still_cancels_and_retrieves_expired_close_child( + monkeypatch, abort_kind +): + entered, ended = asyncio.Event(), asyncio.Event() + secondary = abort_kind("abort failure") + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + try: + await asyncio.Future() + finally: + ended.set() + raise ValueError("late close failure") + + owner = cell_control.Owner(exit_s=0.02) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert isinstance(result, TimeoutError) + assert time.monotonic() - started < 0.2 + assert entered.is_set() + await asyncio.wait_for(ended.wait(), 0.2) + await asyncio.sleep(0) + assert owner._close_task.done() and not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_self_cancelled_close_is_terminal_even_when_abort_fails(monkeypatch, abort_kind): + first = asyncio.CancelledError("close cancelled itself") + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failure") + + async def close(conn, *, end): + calls.append("close") + raise first + + owner = cell_control.Owner(exit_s=60) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert result is first + assert time.monotonic() - started < 0.2 + assert calls == ["close", "abort"] + + +@pytest.mark.parametrize("wait_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_failed_reap_abort_cannot_replace_the_callers_first_failure( + monkeypatch, wait_kind, abort_kind +): + owner = cell_control.Owner() + owner.close_end = time.monotonic() + 0.2 + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failed while reaping") + + async def failed_wait(*args, **kwargs): + raise wait_kind("reap wait interrupted") + + owner.conn = Connection() + child = owner._start(lambda: asyncio.Future()) + monkeypatch.setattr(cell_control.asyncio, "wait", failed_wait) + try: + try: + result = await cell_control._captured(lambda: owner._stop({child})) + except BaseException as error: # noqa: BLE001 — the probe must expose secondary native errors. + result = error + assert result is None, "reaping cannot replace an earlier operation failure" + assert calls == ["abort"] + finally: + child.cancel() + await asyncio.gather(child, return_exceptions=True) From 361bbbb9e3fcbcbc4af3a8ffd8b6b1c38b83a930 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:33:07 +0000 Subject: [PATCH 20/25] Bound placement exchanges and reserve both request bodies Signed-off-by: zlareb1 --- server/cli/cell_place.py | 12 +- server/router/directory.py | 14 +- tests/unit/test_cell_place.py | 4 +- tests/unit/test_cell_place_envelopes.py | 134 +++++++++++++++++ tests/unit/test_cell_place_http_deadline.py | 157 ++++++++++++++++++++ tests/unit/test_router_attach.py | 25 ++++ 6 files changed, 335 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_cell_place_envelopes.py create mode 100644 tests/unit/test_cell_place_http_deadline.py diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index 573e0ff2fc..c0cc15aeb2 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -19,6 +19,9 @@ from server.router.directory import PLACE_MAX_S from server.settings import settings +# The directory statement bound plus connection and client cleanup. +REQUEST_S = PLACE_MAX_S + 10.0 # not a setting: derived + class PlacementRefused(Exception): """A placement cannot complete with this configuration or directory winner.""" @@ -71,7 +74,7 @@ async def _directory( url, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, - timeout=httpx.Timeout(PLACE_MAX_S + 10.0, connect=5.0), + timeout=httpx.Timeout(REQUEST_S, connect=5.0), ) posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) if posted is None: @@ -88,7 +91,7 @@ async def _directory( async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: """Hold the operator lock through directory placement and an acknowledged local commit. - Every local phase, including opening and closing the direct session, has a deadline.""" + Local phases and the complete directory exchange have separate deadlines.""" owner = Owner(exit_s=_bound_s()) async def opening() -> None: @@ -113,7 +116,10 @@ async def initial() -> tuple[str, dict[str, Any]]: async def work() -> dict[str, str]: deployment, fingerprint = await owner.call(initial, total_s=_bound_s()) - await _directory(router, args.cell, args.org, deployment, fingerprint) + await owner.call( + lambda: _directory(router, args.cell, args.org, deployment, fingerprint), + total_s=REQUEST_S, + ) async def local() -> None: conn = owner.conn diff --git a/server/router/directory.py b/server/router/directory.py index 8ee6fff3df..93dadd802f 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -859,12 +859,14 @@ def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> _PLACEMENT_ORG_MAX = 6 * cell_state.ORG_MAX_BYTES # not a setting: derived -def _placement_refusal(cell: str, name: str, value: dict[str, Any]) -> str: - """Why a fingerprint is too large for the cell's live placements ('' when it is not): a - placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's - MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" +def _placement_refusal(cell: str, deployment_id: str, name: str, value: dict[str, Any]) -> str: + """Reserve both live and operator placement bodies with the longest escaped org.""" # The default encoding is the longest: spaces after separators, non-ASCII escaped. - size = len(json.dumps({"org": "", "cell": cell, "fingerprint": value})) + _PLACEMENT_ORG_MAX + envelopes = ( + {"org": "", "cell": cell, "fingerprint": value}, + {"org": "", "deployment_id": deployment_id, "fingerprint": value}, + ) + size = max(len(json.dumps(body)) for body in envelopes) + _PLACEMENT_ORG_MAX if size <= MAX_BODY_BYTES: return "" sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) @@ -890,7 +892,7 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: if value is None: continue if reason := _fingerprint_refusal(found, name, value) or _placement_refusal( - row.cell, name, value + row.cell, row.deployment_id, name, value ): return reason # Compared as JSON, not as Python values, where `false == 0` and `1 == 1.0`: a rerun diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py index 26a06748a1..a8536a27b4 100644 --- a/tests/unit/test_cell_place.py +++ b/tests/unit/test_cell_place.py @@ -486,13 +486,13 @@ async def timed(self, work, *, total_s): return await actual_call(self, work, total_s=total_s) async def directory_call(*args): - # HTTP owns its existing independent transport limit, not a local phase cap. + # The complete HTTP exchange has its independent directory-derived phase. await asyncio.sleep(0.06) monkeypatch.setattr(cell_control.Owner, "call", timed) monkeypatch.setattr(cell_place, "_directory", directory_call) await invoke_place() - assert observed == [0.03, 0.03, 0.03] + assert observed == [0.03, 0.03, cell_place.REQUEST_S, 0.03] conn, _ = local assert conn.committed and conn.closed and not conn.terminated diff --git a/tests/unit/test_cell_place_envelopes.py b/tests/unit/test_cell_place_envelopes.py new file mode 100644 index 0000000000..15dd7d779e --- /dev/null +++ b/tests/unit/test_cell_place_envelopes.py @@ -0,0 +1,134 @@ +"""Every accepted fingerprint fits both placement envelopes for every valid org.""" + +from __future__ import annotations + +import hashlib +import json +from types import SimpleNamespace +from uuid import UUID + +import httpx +import pytest +from starlette.requests import Request + +from server.core import cell +from server.router import app, directory +from server.router.limits import MAX_BODY_BYTES +from server.settings import settings + +DEPLOYMENT = str(UUID(int=1)) +TOKEN = "lco_" + "7" * 128 + + +def fingerprint(): + return cell.as_value( + { + **cell.DEFAULTS, + "deployment_id": DEPLOYMENT, + "public_host": "first.example.test", + "oauth_issuer_host": "first.example.test", + } + ) + + +def envelopes(name, value, org=""): + return ( + {"org": org, "cell": name, "fingerprint": value}, + {"org": org, "deployment_id": DEPLOYMENT, "fingerprint": value}, + ) + + +def attaching(name): + row = SimpleNamespace( + cell=name, + is_first=True, + state="attaching", + mode="recording", + deployment_id=DEPLOYMENT, + current_fp=None, + next_fp=None, + host="first.example.test", + ) + return SimpleNamespace(row=row, other=None, attach_started_at=None, bucket="", bucket_prefix="") + + +def begin(value, position): + return directory.Begin( + DEPLOYMENT, + value if position == "current_fp" else fingerprint(), + value if position == "next_fp" else None, + "a" * 64, + {}, + ) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_reserves_operator_envelope_before_accepting_fingerprint(name, position): + value = fingerprint() + live = envelopes(name, value)[0] + # This fits the previous live-only reservation, but not the operator envelope. + live_size = len(json.dumps(live)) + 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - live_size) + reason = directory.begin_refusal(attaching(name), begin(value, position)) + assert position in reason and str(MAX_BODY_BYTES) in reason + assert "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("size", ["default", "boundary"]) +@pytest.mark.parametrize("org", ["acme", "\x01" * cell.ORG_MAX_BYTES]) +async def test_accepted_fingerprint_fits_both_encodings_and_operator_handler( + monkeypatch, name, position, size, org +): + value = fingerprint() + if size == "boundary": + empty_size = max(len(json.dumps(body)) for body in envelopes(name, value)) + value["fields"]["oidc_client_id"] = "x" * ( + MAX_BODY_BYTES - empty_size - 6 * cell.ORG_MAX_BYTES + ) + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + attempt = begin(value, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + bodies = envelopes(name, value, org) + encoded = [ + (json.dumps(body).encode(), httpx.Request("POST", "http://inert.test", json=body).content) + for body in bodies + ] + assert all(len(raw) <= MAX_BODY_BYTES for pair in encoded for raw in pair) + calls = [] + + async def place(actual_org, actual_name, refusal, *, actor, verify): + calls.append((actual_org, actual_name, actor)) + locked = SimpleNamespace( + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + ) + assert verify(locked) == "" + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + for raw in encoded[1]: + scope = { + "type": "http", + "method": "POST", + "path": f"/cells/v1/operator/cells/{name}/place", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(raw=raw): + return {"type": "http.request", "body": raw, "more_body": False} + + response = await app.Router().operator_place(Request(scope, receive)) + assert response.status_code == 200 + assert json.loads(response.body) == {"org": org, "cell": name, "deployment_id": DEPLOYMENT} + assert calls == [(org, name, "operator")] * 2 diff --git a/tests/unit/test_cell_place_http_deadline.py b/tests/unit/test_cell_place_http_deadline.py new file mode 100644 index 0000000000..79d278903a --- /dev/null +++ b/tests/unit/test_cell_place_http_deadline.py @@ -0,0 +1,157 @@ +"""A complete HTTP exchange cannot outlive its directory-derived owner phase.""" + +from __future__ import annotations + +import asyncio +import json +import time + +import httpcore +import httpx +import pytest +from httpcore._backends.mock import AsyncMockBackend, AsyncMockStream + +from server.cli import cell_control, cell_place +from server.settings import settings +from tests.unit import test_cell_place as helpers + +local = helpers.local +invoke_place = helpers.invoke_place + +_DIRECTORY = cell_place._directory + + +@pytest.fixture +def exchange(local, monkeypatch): + conn, _ = local + events = [] + entered = asyncio.Event() + state = {"mode": "response", "reads": [], "owners": []} + real_close = conn.close + real_call = cell_control.Owner.call + actual_client = httpx.AsyncClient + + async def close(**kwargs): + events.append("pg-close") + await asyncio.sleep(0) + await real_close(**kwargs) + events.append("pg-ack") + + async def timed(owner, work, *, total_s): + if total_s == 0.1: + state["owners"].append(owner) + return await real_call(owner, work, total_s=total_s) + + class Stream(AsyncMockStream): + def __init__(self): + super().__init__([]) + self.headers = False + + async def read(self, max_bytes, timeout=None): # noqa: ASYNC109 — httpcore transport API + if not self.headers: + self.headers = True + if state["mode"] == "response": + return b"HTTP/1.1 200 OK\r\nContent-Length: 100000000\r\n\r\n" + status = b"503 Unavailable" if state["mode"] == "refusal" else b"200 OK" + body = json.dumps( + {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"} + if state["mode"] == "refusal" + else {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + ).encode() + # A timely reply may exceed the separate local phase allowance. + await asyncio.sleep(0.04) + return ( + b"HTTP/1.1 " + + status + + b"\r\nContent-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + entered.set() + state["reads"].append(timeout) + try: + async with asyncio.timeout(timeout): + await asyncio.sleep(0.002) + return b" " + except asyncio.CancelledError: + events.append("http-cancel") + raise + + class Backend(AsyncMockBackend): + async def connect_tcp(self, *args, **kwargs): + return Stream() + + class Client(actual_client): + async def __aexit__(self, *args): + if state["mode"] == "client-exit": + entered.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + events.append("http-cancel") + raise + return await super().__aexit__(*args) + + def client(**kwargs): + transport = httpx.AsyncHTTPTransport() + transport._pool = httpcore.AsyncConnectionPool(network_backend=Backend([])) + return Client(transport=transport, **kwargs) + + conn.close = close + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "REQUEST_S", 0.1, raising=False) + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return conn, events, entered, state + + +@pytest.mark.parametrize("blocking", ["response", "client-exit"]) +@pytest.mark.parametrize("stop", ["deadline", "cancel"]) +async def test_whole_exchange_closes_pg_before_http_cancellation(exchange, blocking, stop): + conn, events, entered, state = exchange + state["mode"] = blocking + started = time.monotonic() + task = asyncio.create_task(invoke_place()) + try: + await asyncio.wait_for(entered.wait(), 0.2) + if stop == "cancel": + task.cancel("operator stop") + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "per-read HTTP timeouts cannot bound a continuing response or client cleanup" + with pytest.raises( + TimeoutError if stop == "deadline" else asyncio.CancelledError + ) as caught: + await task + if stop == "cancel": + assert str(caught.value) == "operator stop" + assert time.monotonic() - started < 0.3 + assert conn.closed and not conn.committed and not conn.terminated + assert not any("INSERT" in call for call in conn.calls) + assert events.index("pg-ack") < events.index("http-cancel") + assert events.count("pg-close") == 1 + if stop == "deadline": + (owner,) = state["owners"] + assert owner.close_end == owner.phase_end + if blocking == "response": + assert state["reads"] and all(value == 0.1 for value in state["reads"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("mode", ["success", "refusal"]) +async def test_timely_actual_http_reply_keeps_local_phase_bounds(exchange, mode): + conn, events, _, state = exchange + state["mode"] = mode + if mode == "refusal": + with pytest.raises(cell_place.PlacementRefused, match="standby stalled"): + await invoke_place() + assert not conn.committed + else: + assert (await invoke_place())["deployment_id"] == "dep-eu-1" + assert conn.committed + assert conn.closed and not conn.terminated + assert events.count("pg-close") == 1 diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index d9d4b9e7b4..4b0ec1f35c 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1456,3 +1456,28 @@ def test_native_compact_batch_with_control_envelope_fits_public_handler(client, assert response.status_code == 200, response.text assert response.json()["placed"] == len(batch) assert set(store.placements) == set(batch) + + +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_refuses_operator_envelope_overflow_before_probe_or_writes( + client: TestClient, store: _Store, listener: _Listener, position: str +) -> None: + deployment = str(UUID(int=1)) + store.cells["eu-1"]["deployment_id"] = deployment + current = cell.as_value({**FIELDS, "deployment_id": deployment}) + grown = cell.as_value({**FIELDS, "deployment_id": deployment, "oidc_client_id": ""}) + live = {"org": "", "cell": "eu-1", "fingerprint": grown} + available = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES + grown["fields"]["oidc_client_id"] = "x" * available + body = ( + _begin(deployment_id=deployment, current_fp=current, **{position: grown}) + if position == "next_fp" + else _begin(deployment_id=deployment, current_fp=grown) + ) + response = _call(client, "POST", "attach/begin", body) + assert response.status_code == 409 + assert position in response.json()["error"] + assert "oidc_client_id" in response.json()["error"] + assert store.statements == directory.ATTACH_READ_STATEMENTS + assert listener.asked == [] and store.audit == [] + assert store.cells["eu-1"]["attach_started_at"] is None From ef7e606efcab77d8086d8e3d860aa6d5f888c840 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:58:38 +0000 Subject: [PATCH 21/25] Reserve native fingerprint defaults in placement bodies Native placement callers serialize every known fingerprint field. Reserve their defaults when a stored fingerprint omits them, while retaining unknown stored fields in the size bound. Signed-off-by: zlareb1 --- server/router/directory.py | 2 + tests/unit/test_cell_place_envelopes.py | 198 ++++++++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/server/router/directory.py b/server/router/directory.py index 93dadd802f..4eca1b74bd 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -861,6 +861,8 @@ def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> def _placement_refusal(cell: str, deployment_id: str, name: str, value: dict[str, Any]) -> str: """Reserve both live and operator placement bodies with the longest escaped org.""" + # Native callers materialize known defaults; retain unknown stored fields too. + value = {**value, "fields": {**cell_state.DEFAULTS, **value["fields"]}} # The default encoding is the longest: spaces after separators, non-ASCII escaped. envelopes = ( {"org": "", "cell": cell, "fingerprint": value}, diff --git a/tests/unit/test_cell_place_envelopes.py b/tests/unit/test_cell_place_envelopes.py index 15dd7d779e..40b89fabda 100644 --- a/tests/unit/test_cell_place_envelopes.py +++ b/tests/unit/test_cell_place_envelopes.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import itertools import json from types import SimpleNamespace from uuid import UUID @@ -132,3 +133,200 @@ async def receive(raw=raw): assert response.status_code == 200 assert json.loads(response.body) == {"org": org, "cell": name, "deployment_id": DEPLOYMENT} assert calls == [(org, name, "operator")] * 2 + + +OMITTABLE_DEFAULTS = ( + "bearer_issuers", + "bucket", + "bucket_prefix", + "configured_orgs", + "end_user_orgs", + "oidc_issuer", + "scim", +) + + +@pytest.fixture +def native_fingerprint(monkeypatch): + configured = SimpleNamespace( + public_base_url="https://first.example.test", + oauth_issuer="", + oidc_issuer="", + oidc_client_id="", + scim_token="", + document_s3_bucket="", + document_s3_prefix="", + oidc_jit_org="", + self_signup="closed", + self_signup_org="", + code_org="", + tracker_org="", + trusted_issuers=lambda: set(), + end_user_issuers=lambda: {}, + ) + monkeypatch.setattr(cell, "settings", configured) + monkeypatch.setattr(cell, "_code_orgs", None) + + def native(client_id): + configured.oidc_client_id = client_id + # Both native placement callers serialize this complete field set. + return cell.as_value(cell.fingerprint(DEPLOYMENT)) + + return native + + +def omitted_values(): + return ( + subset + for count in range(len(OMITTABLE_DEFAULTS) + 1) + for subset in itertools.combinations(OMITTABLE_DEFAULTS, count) + ) + + +def stored_fingerprint(native, omitted, unknown): + return cell.as_value( + { + **{key: value for key, value in native["fields"].items() if key not in omitted}, + **unknown, + } + ) + + +def accepted_row(name, attempt): + return SimpleNamespace( + cell=name, + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + state="serving", + is_first=True, + mode="enforced", + closed=False, + holds=lambda token: True, + ) + + +async def route_response(monkeypatch, name, attempt, body, operator): + calls = [] + row = accepted_row(name, attempt) + + async def place(org, actual_name, refusal, *, actor, verify=None): + assert actual_name == name and refusal(row) == "" + if verify is not None: + assert verify(row) == "" + calls.append((org, actor)) + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + router = app.Router() + + async def authenticated(request, actual_name): + assert actual_name == name + return "inert-cell-credential" + + monkeypatch.setattr(router, "_authenticated", authenticated) + monkeypatch.setattr(router, "placements", SimpleNamespace(admit=lambda name: True)) + raw = httpx.Request("POST", "http://inert.test", json=body).content + scope = { + "type": "http", + "method": "POST", + "path": "/cells/v1/placements", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(): + return {"type": "http.request", "body": raw, "more_body": False} + + handler = router.operator_place if operator else router.place + response = await handler(Request(scope, receive)) + return response, calls, len(raw) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x"}, {"future": "x" * 512}]) +def test_sparse_admission_reserves_matching_native_callers( + native_fingerprint, name, position, unknown +): + org = "\x01" * cell.ORG_MAX_BYTES + for omitted in omitted_values(): + stored = stored_fingerprint(native_fingerprint(""), omitted, unknown) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + stored["fields"]["oidc_client_id"] = client_id + native = native_fingerprint(client_id) + attempt = begin(stored, position) + row = accepted_row(name, attempt) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(row, native) + reason = directory.begin_refusal(attaching(name), attempt) + if not reason: + # Admission must cover what these callers send, even if storage is sparse. + assert all( + len(httpx.Request("POST", "http://inert.test", json=body).content) <= MAX_BODY_BYTES + for body in envelopes(name, native, org) + ), (omitted, unknown) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +async def test_sparse_boundary_refuses_unusable_native_requests( + monkeypatch, native_fingerprint, name, position, operator +): + stored = stored_fingerprint(native_fingerprint(""), OMITTABLE_DEFAULTS, {}) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + stored["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(stored["fields"]["oidc_client_id"]) + attempt = begin(stored, position) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + body = envelopes(name, native, org)[int(operator)] + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + if org == "acme": + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + else: + assert size > MAX_BODY_BYTES + assert response.status_code == (413 if operator else 400) + assert not calls + reason = directory.begin_refusal(attaching(name), attempt) + assert position in reason and "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x" * 512}]) +async def test_accepted_sparse_boundary_serves_native_requests( + monkeypatch, native_fingerprint, name, position, operator, unknown +): + native = native_fingerprint("") + # Unknown stored fields keep their existing reservation, although native callers + # do not emit them. Reserve that representation with all known defaults present. + reserved = cell.as_value({**native["fields"], **unknown}) + empty = max(len(json.dumps(body)) for body in envelopes(name, reserved)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(client_id) + stored = stored_fingerprint(native, OMITTABLE_DEFAULTS, unknown) + attempt = begin(stored, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + body = envelopes(name, native, org)[int(operator)] + assert len(json.dumps(body).encode()) <= MAX_BODY_BYTES + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + stored["fields"]["future"] = "x" * MAX_BODY_BYTES + assert position in directory.begin_refusal(attaching(name), begin(stored, position)) From c8ec1d6b012fbef4885d2356bbf0201f8f1fc502 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:15:33 +0000 Subject: [PATCH 22/25] Describe trigger creation before the attach mode commit Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 0af383cebe..32e81ada9d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -802,9 +802,9 @@ read-only scope, lists any tenancy row the directory does not place. serves past a table whose lock it cannot take, logged with the holders of its locks, or one the role cannot alter, logged as such (the third exception under Invariants). A first cell is `recording` until attach sets `enforced` while it runs, so no boot can - have created them: attach creates them itself with the same pass - (`cell_triggers.ensure`) once it has set `enforced`, and the cell goes `serving` only - once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare + have created them: attach creates them with `cell_triggers.ensure_table`, passing + `recording_first=True`, before step 2 and again before setting `enforced`. + The cell goes `serving` only once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare and gets them at its first migrating boot after prepare, which its attach requires the same way. `loom doctor` names a disabled or altered one in any state, a missing one on an `enforced` cell, and a partitioned table with an `org` column as unsupported, since From 17eee335bb439623e154e77ca3f70d3cf519a10f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:40:33 +0000 Subject: [PATCH 23/25] Keep native HTTP failures through client cleanup Client exit preserves request failures and selected registration refusals. A failed required exit after a successful request still stops the command. Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 15 +- server/cli/cell_control.py | 20 +- server/cli/cell_place.py | 11 +- server/cli/cell_prepare.py | 17 +- .../unit/test_cell_native_http_first_error.py | 263 ++++++++++++++++++ .../unit/test_cell_place_http_first_error.py | 169 +++++++++++ 6 files changed, 482 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_cell_native_http_first_error.py create mode 100644 tests/unit/test_cell_place_http_first_error.py diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index a8a78279fd..8fd6e7ebee 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -90,10 +90,19 @@ async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" try: - async with httpx.AsyncClient( - timeout=_HEALTH_TIMEOUT, transport=self.transport - ) as client: + manager = httpx.AsyncClient(timeout=_HEALTH_TIMEOUT, transport=self.transport) + client = await manager.__aenter__() + try: r = await client.get(f"https://{host}/v1/health/deployment") + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + except (httpx.HTTPError, ValueError): + return "" + # A required close failure after a successful read must reach the owner. + await manager.__aexit__(None, None, None) + try: answer = r.json() except (httpx.HTTPError, ValueError): return "" diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 1966b9e62b..730c36bd42 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import contextlib import json +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -304,16 +306,22 @@ async def request( endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) try: - async with ( - asyncio.timeout(REQUEST_S), - httpx.AsyncClient( + async with asyncio.timeout(REQUEST_S): + manager = httpx.AsyncClient( timeout=cli._OPERATOR_TIMEOUT, trust_env=url.scheme == "https", transport=transport, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) as client, - ): - response = await client.request(method, endpoint, json=body) + ) + client = await manager.__aenter__() + try: + response = await client.request(method, endpoint, json=body) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) except (httpx.HTTPError, TimeoutError) as e: raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index c0cc15aeb2..214c79179f 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -4,6 +4,7 @@ import argparse import asyncio +import contextlib import json import sys from typing import Any @@ -69,13 +70,21 @@ async def _directory( router: httpx.URL, name: str, org: str, deployment: str, fingerprint: dict[str, Any] ) -> None: url = cell_cli.at(router, f"/cells/v1/operator/cells/{name}/place") - async with httpx.AsyncClient(trust_env=router.scheme == "https") as client: + manager = httpx.AsyncClient(trust_env=router.scheme == "https") + client = await manager.__aenter__() + try: answer = await client.post( url, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, timeout=httpx.Timeout(REQUEST_S, connect=5.0), ) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) if posted is None: raise PlacementRefused(cell_cli.not_the_router(answer)) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 6e94ae1058..4310b4c9bb 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,6 +26,7 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets @@ -301,9 +302,11 @@ async def _register( token = settings.router_operator_token url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") deadline = time.monotonic() + args.wait - async with httpx.AsyncClient( + manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport - ) as client: + ) + client = await manager.__aenter__() + try: while True: try: r = await client.post(url, json=body, headers={"Authorization": f"Bearer {token}"}) @@ -318,7 +321,7 @@ async def _register( f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" ) if r.status_code == 200: - return answer + break code = answer.get("code") if code == "BACKEND_UNAVAILABLE": raise Refused( @@ -340,6 +343,14 @@ async def _register( print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) + except BaseException: # Client exit preserves the first request or refusal. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + return answer + def _own_rows( rows: _CellRows, answer: dict[str, Any], every: list[dict[str, Any]] diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py new file mode 100644 index 0000000000..4634e7d69c --- /dev/null +++ b/tests/unit/test_cell_native_http_first_error.py @@ -0,0 +1,263 @@ +"""Native HTTP clients preserve the first outcome through required exit.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.settings import settings + +_CLIENT = httpx.AsyncClient +_BOUNDARIES = ("control", "health", "register") +FIRST_ERRORS = ( + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) +EXIT_ERRORS = ( + RuntimeError, + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) + + +@pytest.fixture +def exchange(monkeypatch): + state = SimpleNamespace( + first=None, + second=None, + phase="request", + close_phase="await", + events=[], + exit_args=None, + entered=asyncio.Event(), + block=False, + answer={"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"}, + status=200, + close_entered=asyncio.Event(), + close_block=False, + ) + + async def handler(request): + state.events.append("request") + state.entered.set() + if state.block: + try: + await asyncio.Future() + except asyncio.CancelledError as error: + state.first = error + raise + if state.first is not None: + raise state.first + return httpx.Response(state.status, json=state.answer) + + class Client(_CLIENT): + async def __aenter__(self): + state.events.append("enter") + if state.phase == "enter": + raise state.first + return await super().__aenter__() + + def __aexit__(self, *args): + state.exit_args = args + state.events.append("exit-call") + if state.close_phase == "construct" and state.second is not None: + raise state.second + + async def close(): + state.events.append("exit-await") + state.close_entered.set() + if state.close_block: + await asyncio.Future() + await super(Client, self).__aexit__(*args) + if state.second is not None: + raise state.second + + return close() + + def client(**kwargs): + state.events.append("construct") + if state.phase == "construct": + raise state.first + kwargs["transport"] = httpx.MockTransport(handler) + return Client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return state + + +async def invoke(boundary): + url = httpx.URL("https://router.test") + if boundary == "control": + return await cell_control.request(url, "eu-1", None, "/claim", {}) + if boundary == "health": + return await cell_attach.Router(url, "eu-1", None).health("eu-1.example.test") + return await cell_prepare._register(argparse.Namespace(cell="eu-1", wait=0), url, {}, None) + + +async def observed(callback): + # Native fatal values must stay in this task rather than reach the event loop. + try: + return "value", await callback() + except BaseException as error: # noqa: BLE001 — compare original native failure objects. + return "error", error + + +def assert_first(boundary, state, outcome): + kind, result = outcome + if boundary == "health" and isinstance(state.first, (httpx.HTTPError, ValueError)): + assert (kind, result) == ("value", "") + elif (boundary == "control" and isinstance(state.first, (httpx.HTTPError, TimeoutError))) or ( + boundary == "register" + and state.phase == "request" + and isinstance(state.first, httpx.HTTPError) + ): + expected = cell_prepare.Refused if boundary == "register" else cell_control.Refused + assert kind == "error" and isinstance(result, expected) + assert type(state.first).__name__ in str(result) + else: + assert kind == "error" and result is state.first + + +def assert_exit_first(boundary, state): + first = state.exit_args[1] + if boundary == "register" and isinstance(state.first, httpx.HTTPError): + assert isinstance(first, cell_prepare.Refused) + else: + assert first is state.first + + +def assert_close_failure(boundary, state, outcome): + kind, result = outcome + if boundary == "control" and isinstance(state.second, (httpx.HTTPError, TimeoutError)): + assert kind == "error" and isinstance(result, cell_control.Refused) + else: + assert kind == "error" and result is state.second + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = ( + None + if boundary == "placement" + else ("dep-eu-1" if boundary == "health" else exchange.answer) + ) + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +@pytest.mark.parametrize("semantic", ("malformed", "backend", "probe", "refusal")) +async def test_registration_selected_refusal_survives_exit( + monkeypatch, exchange, second_kind, close_phase, semantic +): + selected = cell_prepare.Refused("registration-first") + monkeypatch.setattr(cell_prepare, "Refused", lambda message: selected) + exchange.status = 200 if semantic == "malformed" else 503 + exchange.answer = ( + {"unexpected": True} + if semantic == "malformed" + else { + "error": "registration-first", + "code": {"backend": "BACKEND_UNAVAILABLE", "probe": "PROBE_BUSY", "refusal": "OTHER"}[ + semantic + ], + } + ) + exchange.second = second_kind("exit-second") + exchange.close_phase = close_phase + kind, result = await observed(lambda: invoke("register")) + assert kind == "error" and result is selected + assert exchange.exit_args[1] is selected diff --git a/tests/unit/test_cell_place_http_first_error.py b/tests/unit/test_cell_place_http_first_error.py new file mode 100644 index 0000000000..3587eb95ba --- /dev/null +++ b/tests/unit/test_cell_place_http_first_error.py @@ -0,0 +1,169 @@ +"""Placement HTTP failures keep their outcome through client and Postgres cleanup.""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from server.cli import cell_place +from tests.unit import test_cell_native_http_first_error as native +from tests.unit import test_cell_place as place_helpers + +local = place_helpers.local +invoke_place = place_helpers.invoke_place +exchange = native.exchange +observed = native.observed +assert_first = native.assert_first +assert_exit_first = native.assert_exit_first +assert_close_failure = native.assert_close_failure +FIRST_ERRORS = native.FIRST_ERRORS +EXIT_ERRORS = native.EXIT_ERRORS +_BOUNDARIES = ("placement",) +_DIRECTORY = cell_place._directory + + +async def invoke(boundary): + return await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = ( + None + if boundary == "placement" + else ("dep-eu-1" if boundary == "health" else exchange.answer) + ) + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_keeps_http_first_failure_and_closes_pg_once( + local, exchange, monkeypatch, first_kind, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") + kind, result = await observed(invoke_place) + assert kind == "error" + if first_kind is asyncio.CancelledError: + # A cancelled child Task reconstructs this exception when Owner reads it. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_required_http_close_failure_prevents_insert( + local, exchange, monkeypatch, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.second = second_kind("required-exit") + kind, result = await observed(invoke_place) + assert kind == "error" + if second_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.second + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +async def test_owned_placement_successful_http_neighbor_commits(local, exchange, monkeypatch): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + assert await invoke_place() == exchange.answer + assert conn.calls.count("close") == 1 + assert conn.closed and conn.committed and not conn.terminated From ce81164cce96865f546c7c238817dc1ff524ed6e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:05:44 +0000 Subject: [PATCH 24/25] Native nonce cleanup keeps the first failure Prepare and attach attempt nonce deletion without replacing the failure that selected cleanup. Standalone cleanup interruptions still propagate. Remove the unused removal snapshot argument and unreachable success-test branches. Signed-off-by: zlareb1 --- server/cli/cell.py | 3 +- server/cli/cell_attach.py | 15 +- server/cli/cell_prepare.py | 6 +- .../unit/test_cell_native_http_first_error.py | 6 +- .../test_cell_native_nonce_first_error.py | 269 ++++++++++++++++++ .../unit/test_cell_place_http_first_error.py | 7 +- 6 files changed, 289 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_cell_native_nonce_first_error.py diff --git a/server/cli/cell.py b/server/cli/cell.py index a6fb4e21e5..d583fb48dc 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -349,7 +349,7 @@ def _cmd_remove(args: argparse.Namespace) -> int: print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 try: - answer = asyncio.run(_remove_locked(args, router, row)) + answer = asyncio.run(_remove_locked(args, router)) except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) return 1 @@ -360,7 +360,6 @@ def _cmd_remove(args: argparse.Namespace) -> int: async def _remove_locked( args: argparse.Namespace, router: httpx.URL, - row: dict[str, Any], transport: httpx.AsyncBaseTransport | None = None, ) -> dict[str, Any]: from server.cli import cell_control diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8fd6e7ebee..821a77448b 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -376,7 +376,11 @@ async def begin(self) -> None: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) # -- the scans --------------------------------------------------------------------- @@ -542,11 +546,16 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.mutate( + answer = await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) + return answer # -- the run ----------------------------------------------------------------------- diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4310b4c9bb..517a461840 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -513,7 +513,11 @@ async def _prepare_locked( try: await _write_nonces(ch, body["nonces"]) answer = await _register(args, router, body, transport) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(ch, body["nonces"]) + raise + else: await _delete_nonces(ch, body["nonces"]) # The processes may have changed while the probe was retried: judge the ones live now. beats = await _read_beats(ch) diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py index 4634e7d69c..c9b61bd146 100644 --- a/tests/unit/test_cell_native_http_first_error.py +++ b/tests/unit/test_cell_native_http_first_error.py @@ -180,11 +180,7 @@ async def test_required_client_exit_failure_cannot_report_success( async def test_successful_client_neighbor(exchange, boundary): kind, result = await observed(lambda: invoke(boundary)) assert kind == "value" - expected = ( - None - if boundary == "placement" - else ("dep-eu-1" if boundary == "health" else exchange.answer) - ) + expected = "dep-eu-1" if boundary == "health" else exchange.answer assert result == expected assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] diff --git a/tests/unit/test_cell_native_nonce_first_error.py b/tests/unit/test_cell_native_nonce_first_error.py new file mode 100644 index 0000000000..94254c3d96 --- /dev/null +++ b/tests/unit/test_cell_native_nonce_first_error.py @@ -0,0 +1,269 @@ +"""Native callers retain their first outcome while deleting their probe nonces.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.core import leases, object_store, pg +from server.settings import settings +from tests.unit import test_cell_native_http_first_error as native_http +from tests.unit.test_cell_attach import FakeConn +from tests.unit.test_cell_control import snapshot + +exchange = native_http.exchange +observed = native_http.observed +FIRST_ERRORS = native_http.FIRST_ERRORS +CALLERS = ("prepare", "begin", "serving") +FATAL_ERRORS = (asyncio.CancelledError, SystemExit, KeyboardInterrupt) +CLEANUP_PHASES = ("postgres", "clickhouse", "bucket") +URL = httpx.URL("http://router.test") + + +@pytest.fixture +def nonce_edges(monkeypatch, exchange): + state = SimpleNamespace( + secondary=None, + phase="postgres", + write_first=False, + events=[], + normalized=[], + cleanup_entered=asyncio.Event(), + cleanup_block=False, + ) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": "bucket"} + + async def deletion(phase): + state.events.append(phase) + if phase == state.phase: + state.cleanup_entered.set() + if state.cleanup_block: + await asyncio.Future() + if state.secondary is not None: + raise state.secondary + + async def postgres(*args): + await deletion("postgres") + + async def clickhouse(*args): + assert args[0] == leases.LOCK_RELEASE + await deletion("clickhouse") + + async def read_object(location): + return b"bucket" + + async def bucket(*args): + await deletion("bucket") + + async def write(*args): + if state.write_first: + raise exchange.first + + monkeypatch.setattr(pg, "execute", postgres) + monkeypatch.setattr(object_store, "root_prefix", lambda: "") + monkeypatch.setattr(object_store, "read", read_object) + monkeypatch.setattr(object_store, "delete", bucket) + monkeypatch.setattr(cell_prepare, "_new_nonces", lambda: nonces.copy()) + monkeypatch.setattr(cell_prepare, "_write_nonces", write) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "bucket-test") + state.ch = SimpleNamespace(execute=clickhouse) + return state + + +@pytest.fixture +def native_call(monkeypatch, nonce_edges, exchange): + state = nonce_edges + row = {**snapshot(), "is_first": False, "state": "attaching", "mode": "enforced"} + exchange.answer = row.copy() + conn = FakeConn() + rows = SimpleNamespace(deployment_id="dep", first=None, tenancy_rows=0) + monkeypatch.setattr(cell_prepare, "_read_rows", AsyncMock(return_value=rows)) + monkeypatch.setattr(cell_prepare, "_read_beats", AsyncMock(return_value=[])) + monkeypatch.setattr(cell_prepare, "_beat_fields", lambda *args: [{}]) + monkeypatch.setattr(cell_prepare, "_state_rows", lambda rows: 0) + monkeypatch.setattr(cell_prepare, "_prefix_objects", AsyncMock(return_value=0)) + monkeypatch.setattr(cell_prepare, "_answer_problem", lambda *args: "") + monkeypatch.setattr(cell_prepare, "_own_rows", lambda *args: ({}, None)) + state.write_rows = AsyncMock(return_value=[]) + monkeypatch.setattr(cell_prepare, "_write_rows", state.write_rows) + monkeypatch.setattr(cell_control, "claim", AsyncMock(side_effect=lambda conn, call, row: row)) + args = argparse.Namespace( + cell="first", + wait=0, + host="first.example.test", + credential_sha256="hash", + probe_url="", + ch_allow_user=[], + ch_allow_user_directory=[], + ch_allow_object=[], + ch_allow_replica=[], + pg_allow_role=[], + ) + + async def mutate(path, body): + return await cell_control.request(URL, "first", None, path, body) + + run = SimpleNamespace( + ch=state.ch, args=args, row=row, own_fp=({}, None), keys_fp="key", mutate=mutate + ) + + async def invoke(caller, owned): + owner = cell_control.Owner(exit_s=0.5) + direct = SimpleNamespace( + conn=conn, transaction=lambda work: work(), call=lambda work, **kw: work() + ) + + async def work(): + if caller == "prepare": + return await cell_prepare._prepare_locked( + state.ch, args, URL, None, owner if owned else direct + ) + return await getattr(cell_attach.Run, caller)(run) + + async def opening(): + owner.conn = conn + + return await owner.run(opening, work, opening_s=0.5) if owned else await work() + + state.conn = conn + state.invoke = invoke + return state + + +@pytest.fixture(autouse=True) +def selected_normalization(monkeypatch, nonce_edges): + original_prepare = cell_prepare.Refused + original_control = cell_control._refused + + class PrepareRefused(original_prepare): + def __init__(self, message): + super().__init__(message) + nonce_edges.normalized.append(self) + + def control_refused(message): + selected = original_control(message) + nonce_edges.normalized.append(selected) + return selected + + monkeypatch.setattr(cell_prepare, "Refused", PrepareRefused) + monkeypatch.setattr(cell_control, "_refused", control_refused) + + +def assert_first(state, exchange, caller, first_kind, owned, outcome): + kind, result = outcome + assert kind == "error" + normalized = not state.write_first and ( + (caller == "prepare" and issubclass(first_kind, httpx.HTTPError)) + or (caller != "prepare" and issubclass(first_kind, (httpx.HTTPError, TimeoutError))) + ) + if normalized: + assert state.normalized and result is state.normalized[0] + elif owned and first_kind is asyncio.CancelledError: + # asyncio reconstructs cancellation when Owner retrieves its child Task. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert state.write_rows.await_count == 0 + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +@pytest.mark.parametrize("http_exit_failure", (False, True)) +async def test_http_first_outcome_survives_native_nonce_cleanup( + native_call, exchange, caller, owned, first_kind, cleanup_kind, phase, http_exit_failure +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-second") if cleanup_kind is not None else None + exchange.first = first_kind("request-first") + exchange.second = KeyboardInterrupt("http-exit-second") if http_exit_failure else None + assert_first( + state, + exchange, + caller, + first_kind, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert phase in state.events and len(exchange.exit_args) == 3 + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, ValueError, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +async def test_standalone_nonce_cleanup_keeps_cancellation_and_success_neighbors( + native_call, exchange, caller, owned, cleanup_kind, phase +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-only") if cleanup_kind is not None else None + kind, result = await observed(lambda: state.invoke(caller, owned)) + if cleanup_kind in FATAL_ERRORS: + assert kind == "error" + if owned and cleanup_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is state.secondary + assert state.write_rows.await_count == 0 + else: + assert kind == "value" + assert result == ( + 0 if caller == "prepare" else None if caller == "begin" else exchange.answer + ) + assert state.events == list(CLEANUP_PHASES) + assert state.write_rows.await_count == int(caller == "prepare") + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", FATAL_ERRORS) +async def test_nonce_write_failure_stays_first(native_call, exchange, caller, owned, cleanup_kind): + state = native_call + state.write_first = True + state.secondary = cleanup_kind("nonce-second") + exchange.first = ValueError("write-first") + assert_first( + state, + exchange, + caller, + ValueError, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert exchange.events == [] and state.events == ["postgres"] + + +@pytest.mark.parametrize("caller", CALLERS) +async def test_repeated_cancellation_keeps_first_stop_through_nonce_deletion( + native_call, exchange, caller +): + state = native_call + state.cleanup_block = True + exchange.block = True + task = asyncio.create_task(observed(lambda: state.invoke(caller, False))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(state.cleanup_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + assert state.write_rows.await_count == 0 + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_place_http_first_error.py b/tests/unit/test_cell_place_http_first_error.py index 3587eb95ba..b8120702c9 100644 --- a/tests/unit/test_cell_place_http_first_error.py +++ b/tests/unit/test_cell_place_http_first_error.py @@ -62,12 +62,7 @@ async def test_required_client_exit_failure_cannot_report_success( async def test_successful_client_neighbor(exchange, boundary): kind, result = await observed(lambda: invoke(boundary)) assert kind == "value" - expected = ( - None - if boundary == "placement" - else ("dep-eu-1" if boundary == "health" else exchange.answer) - ) - assert result == expected + assert result is None assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] From 5762c94c60ec1b7d1d9811174e5b8bdb3a39fc01 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:11:09 +0000 Subject: [PATCH 25/25] Placement fence tests name the cell actor Keep queued-placement and backout assertions on the required directory.place call contract. Signed-off-by: zlareb1 --- tests/integration/test_cell_control_fence.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index e14eab1220..14d469d743 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -468,7 +468,7 @@ async def wait_for_lock(key): "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) # The placement has its org lock and is queued at the cell row before claim. for _ in range(100): @@ -502,7 +502,7 @@ async def wait_for_lock(key): "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) await wait_for_lock(directory._PLACE_LOCK) await cell_control.backout(_owner(local_admin), call, original) @@ -661,7 +661,9 @@ async def call(path, body): ) # Actually attempt placement before the acknowledged close returns to cleanup. try: - await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + await directory.place( + "between-close-and-cleanup", "first", app.placement_refusal, actor="cell:first" + ) except directory.PlacementRefused as exc: placement_error = str(exc) return answer